1 // SPDX-License-Identifier: GPL-2.0-only OR MIT
3 * Bluetooth HCI driver for Broadcom 4377/4378/4387 devices attached via PCIe
5 * Copyright (C) The Asahi Linux Contributors
8 #include <linux/async.h>
9 #include <linux/bitfield.h>
10 #include <linux/completion.h>
11 #include <linux/dma-mapping.h>
12 #include <linux/dmi.h>
13 #include <linux/firmware.h>
14 #include <linux/module.h>
15 #include <linux/msi.h>
17 #include <linux/pci.h>
18 #include <linux/printk.h>
20 #include <asm/unaligned.h>
22 #include <net/bluetooth/bluetooth.h>
23 #include <net/bluetooth/hci_core.h>
31 #define BCM4377_DEVICE_ID 0x5fa0
32 #define BCM4378_DEVICE_ID 0x5f69
33 #define BCM4387_DEVICE_ID 0x5f71
35 #define BCM4377_TIMEOUT 1000
38 * These devices only support DMA transactions inside a 32bit window
39 * (possibly to avoid 64 bit arithmetic). The window size cannot exceed
40 * 0xffffffff but is always aligned down to the previous 0x200 byte boundary
41 * which effectively limits the window to [start, start+0xfffffe00].
42 * We just limit the DMA window to [0, 0xfffffe00] to make sure we don't
43 * run into this limitation.
45 #define BCM4377_DMA_MASK 0xfffffe00
47 #define BCM4377_PCIECFG_BAR0_WINDOW1 0x80
48 #define BCM4377_PCIECFG_BAR0_WINDOW2 0x70
49 #define BCM4377_PCIECFG_BAR0_CORE2_WINDOW1 0x74
50 #define BCM4377_PCIECFG_BAR0_CORE2_WINDOW2 0x78
51 #define BCM4377_PCIECFG_BAR2_WINDOW 0x84
53 #define BCM4377_PCIECFG_BAR0_CORE2_WINDOW1_DEFAULT 0x18011000
54 #define BCM4377_PCIECFG_BAR2_WINDOW_DEFAULT 0x19000000
56 #define BCM4377_PCIECFG_SUBSYSTEM_CTRL 0x88
58 #define BCM4377_BAR0_FW_DOORBELL 0x140
59 #define BCM4377_BAR0_RTI_CONTROL 0x144
61 #define BCM4377_BAR0_SLEEP_CONTROL 0x150
62 #define BCM4377_BAR0_SLEEP_CONTROL_UNQUIESCE 0
63 #define BCM4377_BAR0_SLEEP_CONTROL_AWAKE 2
64 #define BCM4377_BAR0_SLEEP_CONTROL_QUIESCE 3
66 #define BCM4377_BAR0_DOORBELL 0x174
67 #define BCM4377_BAR0_DOORBELL_VALUE GENMASK(31, 16)
68 #define BCM4377_BAR0_DOORBELL_IDX GENMASK(15, 8)
69 #define BCM4377_BAR0_DOORBELL_RING BIT(5)
71 #define BCM4377_BAR0_HOST_WINDOW_LO 0x590
72 #define BCM4377_BAR0_HOST_WINDOW_HI 0x594
73 #define BCM4377_BAR0_HOST_WINDOW_SIZE 0x598
75 #define BCM4377_BAR2_BOOTSTAGE 0x200454
77 #define BCM4377_BAR2_FW_LO 0x200478
78 #define BCM4377_BAR2_FW_HI 0x20047c
79 #define BCM4377_BAR2_FW_SIZE 0x200480
81 #define BCM4377_BAR2_CONTEXT_ADDR_LO 0x20048c
82 #define BCM4377_BAR2_CONTEXT_ADDR_HI 0x200450
84 #define BCM4377_BAR2_RTI_STATUS 0x20045c
85 #define BCM4377_BAR2_RTI_WINDOW_LO 0x200494
86 #define BCM4377_BAR2_RTI_WINDOW_HI 0x200498
87 #define BCM4377_BAR2_RTI_WINDOW_SIZE 0x20049c
89 #define BCM4377_OTP_SIZE 0xe0
90 #define BCM4377_OTP_SYS_VENDOR 0x15
91 #define BCM4377_OTP_CIS 0x80
92 #define BCM4377_OTP_VENDOR_HDR 0x00000008
93 #define BCM4377_OTP_MAX_PARAM_LEN 16
95 #define BCM4377_N_TRANSFER_RINGS 9
96 #define BCM4377_N_COMPLETION_RINGS 6
98 #define BCM4377_MAX_RING_SIZE 256
100 #define BCM4377_MSGID_GENERATION GENMASK(15, 8)
101 #define BCM4377_MSGID_ID GENMASK(7, 0)
103 #define BCM4377_RING_N_ENTRIES 128
105 #define BCM4377_CONTROL_MSG_SIZE 0x34
106 #define BCM4377_XFER_RING_MAX_INPLACE_PAYLOAD_SIZE (4 * 0xff)
108 #define MAX_ACL_PAYLOAD_SIZE (HCI_MAX_FRAME_SIZE + HCI_ACL_HDR_SIZE)
109 #define MAX_SCO_PAYLOAD_SIZE (HCI_MAX_SCO_SIZE + HCI_SCO_HDR_SIZE)
110 #define MAX_EVENT_PAYLOAD_SIZE (HCI_MAX_EVENT_SIZE + HCI_EVENT_HDR_SIZE)
112 enum bcm4377_otp_params_type {
113 BCM4377_OTP_BOARD_PARAMS,
114 BCM4377_OTP_CHIP_PARAMS
117 enum bcm4377_transfer_ring_id {
118 BCM4377_XFER_RING_CONTROL = 0,
119 BCM4377_XFER_RING_HCI_H2D = 1,
120 BCM4377_XFER_RING_HCI_D2H = 2,
121 BCM4377_XFER_RING_SCO_H2D = 3,
122 BCM4377_XFER_RING_SCO_D2H = 4,
123 BCM4377_XFER_RING_ACL_H2D = 5,
124 BCM4377_XFER_RING_ACL_D2H = 6,
127 enum bcm4377_completion_ring_id {
128 BCM4377_ACK_RING_CONTROL = 0,
129 BCM4377_ACK_RING_HCI_ACL = 1,
130 BCM4377_EVENT_RING_HCI_ACL = 2,
131 BCM4377_ACK_RING_SCO = 3,
132 BCM4377_EVENT_RING_SCO = 4,
135 enum bcm4377_doorbell {
136 BCM4377_DOORBELL_CONTROL = 0,
137 BCM4377_DOORBELL_HCI_H2D = 1,
138 BCM4377_DOORBELL_HCI_D2H = 2,
139 BCM4377_DOORBELL_ACL_H2D = 3,
140 BCM4377_DOORBELL_ACL_D2H = 4,
141 BCM4377_DOORBELL_SCO = 6,
145 * Transfer ring entry
147 * flags: Flags to indicate if the payload is appended or mapped
148 * len: Payload length
149 * payload: Optional payload DMA address
150 * id: Message id to recognize the answer in the completion ring entry
152 struct bcm4377_xfer_ring_entry {
153 #define BCM4377_XFER_RING_FLAG_PAYLOAD_MAPPED BIT(0)
154 #define BCM4377_XFER_RING_FLAG_PAYLOAD_IN_FOOTER BIT(1)
162 static_assert(sizeof(struct bcm4377_xfer_ring_entry) == 0x10);
165 * Completion ring entry
167 * flags: Flags to indicate if the payload is appended or mapped. If the payload
168 * is mapped it can be found in the buffer of the corresponding transfer
170 * ring_id: Transfer ring ID which required this message
171 * msg_id: Message ID specified in transfer ring entry
172 * len: Payload length
174 struct bcm4377_completion_ring_entry {
182 static_assert(sizeof(struct bcm4377_completion_ring_entry) == 0x10);
184 enum bcm4377_control_message_type {
185 BCM4377_CONTROL_MSG_CREATE_XFER_RING = 1,
186 BCM4377_CONTROL_MSG_CREATE_COMPLETION_RING = 2,
187 BCM4377_CONTROL_MSG_DESTROY_XFER_RING = 3,
188 BCM4377_CONTROL_MSG_DESTROY_COMPLETION_RING = 4,
192 * Control message used to create a completion ring
194 * msg_type: Must be BCM4377_CONTROL_MSG_CREATE_COMPLETION_RING
195 * header_size: Unknown, but probably reserved space in front of the entry
196 * footer_size: Number of 32 bit words reserved for payloads after the entry
197 * id/id_again: Completion ring index
198 * ring_iova: DMA address of the ring buffer
199 * n_elements: Number of elements inside the ring buffer
200 * msi: MSI index, doesn't work for all rings though and should be zero
201 * intmod_delay: Unknown delay
202 * intmod_bytes: Unknown
204 struct bcm4377_create_completion_ring_msg {
222 static_assert(sizeof(struct bcm4377_create_completion_ring_msg) ==
223 BCM4377_CONTROL_MSG_SIZE);
226 * Control ring message used to destroy a completion ring
228 * msg_type: Must be BCM4377_CONTROL_MSG_DESTROY_COMPLETION_RING
229 * ring_id: Completion ring to be destroyed
231 struct bcm4377_destroy_completion_ring_msg {
237 static_assert(sizeof(struct bcm4377_destroy_completion_ring_msg) ==
238 BCM4377_CONTROL_MSG_SIZE);
241 * Control message used to create a transfer ring
243 * msg_type: Must be BCM4377_CONTROL_MSG_CREATE_XFER_RING
244 * header_size: Number of 32 bit words reserved for unknown content before the
246 * footer_size: Number of 32 bit words reserved for payloads after the entry
247 * ring_id/ring_id_again: Transfer ring index
248 * ring_iova: DMA address of the ring buffer
249 * n_elements: Number of elements inside the ring buffer
250 * completion_ring_id: Completion ring index for acknowledgements and events
251 * doorbell: Doorbell index used to notify device of new entries
252 * flags: Transfer ring flags
253 * - virtual: set if there is no associated shared memory and only the
254 * corresponding completion ring is used
255 * - sync: only set for the SCO rings
257 struct bcm4377_create_transfer_ring_msg {
263 __le16 ring_id_again;
267 __le16 completion_ring_id;
269 #define BCM4377_XFER_RING_FLAG_VIRTUAL BIT(7)
270 #define BCM4377_XFER_RING_FLAG_SYNC BIT(8)
274 static_assert(sizeof(struct bcm4377_create_transfer_ring_msg) ==
275 BCM4377_CONTROL_MSG_SIZE);
278 * Control ring message used to destroy a transfer ring
280 * msg_type: Must be BCM4377_CONTROL_MSG_DESTROY_XFER_RING
281 * ring_id: Transfer ring to be destroyed
283 struct bcm4377_destroy_transfer_ring_msg {
289 static_assert(sizeof(struct bcm4377_destroy_transfer_ring_msg) ==
290 BCM4377_CONTROL_MSG_SIZE);
293 * "Converged IPC" context struct used to make the device aware of all other
294 * shared memory structures. A pointer to this structure is configured inside a
297 * version: Protocol version, must be 2.
298 * size: Size of this structure, must be 0x68.
299 * enabled_caps: Enabled capabilities. Unknown bitfield but should be 2.
300 * peripheral_info_addr: DMA address for a 0x20 buffer to which the device will
301 * write unknown contents
302 * {completion,xfer}_ring_{tails,heads}_addr: DMA pointers to ring heads/tails
303 * n_completion_rings: Number of completion rings, the firmware only works if
304 * this is set to BCM4377_N_COMPLETION_RINGS.
305 * n_xfer_rings: Number of transfer rings, the firmware only works if
306 * this is set to BCM4377_N_TRANSFER_RINGS.
307 * control_completion_ring_addr: Control completion ring buffer DMA address
308 * control_xfer_ring_addr: Control transfer ring buffer DMA address
309 * control_xfer_ring_n_entries: Number of control transfer ring entries
310 * control_completion_ring_n_entries: Number of control completion ring entries
311 * control_xfer_ring_doorbell: Control transfer ring doorbell
312 * control_completion_ring_doorbell: Control completion ring doorbell,
313 * must be set to 0xffff
314 * control_xfer_ring_msi: Control completion ring MSI index, must be 0
315 * control_completion_ring_msi: Control completion ring MSI index, must be 0.
316 * control_xfer_ring_header_size: Number of 32 bit words reserved in front of
317 * every control transfer ring entry
318 * control_xfer_ring_footer_size: Number of 32 bit words reserved after every
319 * control transfer ring entry
320 * control_completion_ring_header_size: Number of 32 bit words reserved in front
321 * of every control completion ring entry
322 * control_completion_ring_footer_size: Number of 32 bit words reserved after
323 * every control completion ring entry
324 * scratch_pad: Optional scratch pad DMA address
325 * scratch_pad_size: Scratch pad size
327 struct bcm4377_context {
332 __le64 peripheral_info_addr;
334 /* ring heads and tails */
335 __le64 completion_ring_heads_addr;
336 __le64 xfer_ring_tails_addr;
337 __le64 completion_ring_tails_addr;
338 __le64 xfer_ring_heads_addr;
339 __le16 n_completion_rings;
342 /* control ring configuration */
343 __le64 control_completion_ring_addr;
344 __le64 control_xfer_ring_addr;
345 __le16 control_xfer_ring_n_entries;
346 __le16 control_completion_ring_n_entries;
347 __le16 control_xfer_ring_doorbell;
348 __le16 control_completion_ring_doorbell;
349 __le16 control_xfer_ring_msi;
350 __le16 control_completion_ring_msi;
351 u8 control_xfer_ring_header_size;
352 u8 control_xfer_ring_footer_size;
353 u8 control_completion_ring_header_size;
354 u8 control_completion_ring_footer_size;
360 __le32 scratch_pad_size;
364 static_assert(sizeof(struct bcm4377_context) == 0x68);
366 #define BCM4378_CALIBRATION_CHUNK_SIZE 0xe6
367 struct bcm4378_hci_send_calibration_cmd {
370 u8 data[BCM4378_CALIBRATION_CHUNK_SIZE];
373 #define BCM4378_PTB_CHUNK_SIZE 0xcf
374 struct bcm4378_hci_send_ptb_cmd {
376 u8 data[BCM4378_PTB_CHUNK_SIZE];
380 * Shared memory structure used to store the ring head and tail pointers.
382 struct bcm4377_ring_state {
383 __le16 completion_ring_head[BCM4377_N_COMPLETION_RINGS];
384 __le16 completion_ring_tail[BCM4377_N_COMPLETION_RINGS];
385 __le16 xfer_ring_head[BCM4377_N_TRANSFER_RINGS];
386 __le16 xfer_ring_tail[BCM4377_N_TRANSFER_RINGS];
390 * A transfer ring can be used in two configurations:
391 * 1) Send control or HCI messages to the device which are then acknowledged
392 * in the corresponding completion ring
393 * 2) Receiving HCI frames from the devices. In this case the transfer ring
394 * itself contains empty messages that are acknowledged once data is
395 * available from the device. If the payloads fit inside the footers
396 * of the completion ring the transfer ring can be configured to be
397 * virtual such that it has no ring buffer.
399 * ring_id: ring index hardcoded in the firmware
400 * doorbell: doorbell index to notify device of new entries
401 * payload_size: optional in-place payload size
402 * mapped_payload_size: optional out-of-place payload size
403 * completion_ring: index of corresponding completion ring
404 * n_entries: number of entries inside this ring
405 * generation: ring generation; incremented on hci_open to detect stale messages
406 * sync: set to true for SCO rings
407 * virtual: set to true if this ring has no entries and is just required to
408 * setup a corresponding completion ring for device->host messages
409 * d2h_buffers_only: set to true if this ring is only used to provide large
410 * buffers used by device->host messages in the completion
412 * allow_wait: allow to wait for messages to be acknowledged
413 * enabled: true once the ring has been created and can be used
414 * ring: ring buffer for entries (struct bcm4377_xfer_ring_entry)
415 * ring_dma: DMA address for ring entry buffer
416 * payloads: payload buffer for mapped_payload_size payloads
417 * payloads_dma:DMA address for payload buffer
418 * events: pointer to array of completions if waiting is allowed
419 * msgids: bitmap to keep track of used message ids
420 * lock: Spinlock to protect access to ring structurs used in the irq handler
422 struct bcm4377_transfer_ring {
423 enum bcm4377_transfer_ring_id ring_id;
424 enum bcm4377_doorbell doorbell;
426 size_t mapped_payload_size;
433 bool d2h_buffers_only;
441 dma_addr_t payloads_dma;
443 struct completion **events;
444 DECLARE_BITMAP(msgids, BCM4377_MAX_RING_SIZE);
449 * A completion ring can be either used to either acknowledge messages sent in
450 * the corresponding transfer ring or to receive messages associated with the
451 * transfer ring. When used to receive messages the transfer ring either
452 * has no ring buffer and is only advanced ("virtual transfer ring") or it
453 * only contains empty DMA buffers to be used for the payloads.
455 * ring_id: completion ring id, hardcoded in firmware
456 * payload_size: optional payload size after each entry
457 * delay: unknown delay
458 * n_entries: number of entries in this ring
459 * enabled: true once the ring has been created and can be used
460 * ring: ring buffer for entries (struct bcm4377_completion_ring_entry)
461 * ring_dma: DMA address of ring buffer
462 * transfer_rings: bitmap of corresponding transfer ring ids
464 struct bcm4377_completion_ring {
465 enum bcm4377_completion_ring_id ring_id;
474 unsigned long transfer_rings;
480 * Chip-specific configuration struct
482 * id: Chip id (e.g. 0x4377 for BCM4377)
483 * otp_offset: Offset to the start of the OTP inside BAR0
484 * bar0_window1: Backplane address mapped to the first window in BAR0
485 * bar0_window2: Backplane address mapped to the second window in BAR0
486 * bar0_core2_window2: Optional backplane address mapped to the second core's
487 * second window in BAR0
488 * has_bar0_core2_window2: Set to true if this chip requires the second core's
489 * second window to be configured
490 * clear_pciecfg_subsystem_ctrl_bit19: Set to true if bit 19 in the
491 * vendor-specific subsystem control
492 * register has to be cleared
493 * disable_aspm: Set to true if ASPM must be disabled due to hardware errata
494 * broken_ext_scan: Set to true if the chip erroneously claims to support
496 * broken_mws_transport_config: Set to true if the chip erroneously claims to
497 * support MWS Transport Configuration
498 * send_calibration: Optional callback to send calibration data
499 * send_ptb: Callback to send "PTB" regulatory/calibration data
508 u32 bar0_core2_window2;
510 unsigned long has_bar0_core2_window2 : 1;
511 unsigned long clear_pciecfg_subsystem_ctrl_bit19 : 1;
512 unsigned long disable_aspm : 1;
513 unsigned long broken_ext_scan : 1;
514 unsigned long broken_mws_transport_config : 1;
515 unsigned long broken_le_coded : 1;
517 int (*send_calibration)(struct bcm4377_data *bcm4377);
518 int (*send_ptb)(struct bcm4377_data *bcm4377,
519 const struct firmware *fw);
522 static const struct bcm4377_hw bcm4377_hw_variants[];
523 static const struct dmi_system_id bcm4377_dmi_board_table[];
526 * Private struct associated with each device containing global state
528 * pdev: Pointer to associated struct pci_dev
529 * hdev: Pointer to associated strucy hci_dev
530 * bar0: iomem pointing to BAR0
531 * bar1: iomem pointing to BAR2
532 * bootstage: Current value of the bootstage
533 * rti_status: Current "RTI" status value
534 * hw: Pointer to chip-specific struct bcm4377_hw
535 * taurus_cal_blob: "Taurus" calibration blob used for some chips
536 * taurus_cal_size: "Taurus" calibration blob size
537 * taurus_beamforming_cal_blob: "Taurus" beamforming calibration blob used for
539 * taurus_beamforming_cal_size: "Taurus" beamforming calibration blob size
540 * stepping: Chip stepping read from OTP; used for firmware selection
541 * vendor: Antenna vendor read from OTP; used for firmware selection
542 * board_type: Board type from FDT or DMI match; used for firmware selection
543 * event: Event for changed bootstage or rti_status; used for booting firmware
544 * ctx: "Converged IPC" context
545 * ctx_dma: "Converged IPC" context DMA address
546 * ring_state: Shared memory buffer containing ring head and tail indexes
547 * ring_state_dma: DMA address for ring_state
548 * {control,hci_acl,sco}_ack_ring: Completion rings used to acknowledge messages
549 * {hci_acl,sco}_event_ring: Completion rings used for device->host messages
550 * control_h2d_ring: Transfer ring used for control messages
551 * {hci,sco,acl}_h2d_ring: Transfer ring used to transfer HCI frames
552 * {hci,sco,acl}_d2h_ring: Transfer ring used to receive HCI frames in the
553 * corresponding completion ring
555 struct bcm4377_data {
556 struct pci_dev *pdev;
557 struct hci_dev *hdev;
565 const struct bcm4377_hw *hw;
567 const void *taurus_cal_blob;
569 const void *taurus_beamforming_cal_blob;
570 int taurus_beamforming_cal_size;
572 char stepping[BCM4377_OTP_MAX_PARAM_LEN];
573 char vendor[BCM4377_OTP_MAX_PARAM_LEN];
574 const char *board_type;
576 struct completion event;
578 struct bcm4377_context *ctx;
581 struct bcm4377_ring_state *ring_state;
582 dma_addr_t ring_state_dma;
585 * The HCI and ACL rings have to be merged because this structure is
586 * hardcoded in the firmware.
588 struct bcm4377_completion_ring control_ack_ring;
589 struct bcm4377_completion_ring hci_acl_ack_ring;
590 struct bcm4377_completion_ring hci_acl_event_ring;
591 struct bcm4377_completion_ring sco_ack_ring;
592 struct bcm4377_completion_ring sco_event_ring;
594 struct bcm4377_transfer_ring control_h2d_ring;
595 struct bcm4377_transfer_ring hci_h2d_ring;
596 struct bcm4377_transfer_ring hci_d2h_ring;
597 struct bcm4377_transfer_ring sco_h2d_ring;
598 struct bcm4377_transfer_ring sco_d2h_ring;
599 struct bcm4377_transfer_ring acl_h2d_ring;
600 struct bcm4377_transfer_ring acl_d2h_ring;
603 static void bcm4377_ring_doorbell(struct bcm4377_data *bcm4377, u8 doorbell,
608 db |= FIELD_PREP(BCM4377_BAR0_DOORBELL_VALUE, val);
609 db |= FIELD_PREP(BCM4377_BAR0_DOORBELL_IDX, doorbell);
610 db |= BCM4377_BAR0_DOORBELL_RING;
612 dev_dbg(&bcm4377->pdev->dev, "write %d to doorbell #%d (0x%x)\n", val,
614 iowrite32(db, bcm4377->bar0 + BCM4377_BAR0_DOORBELL);
617 static int bcm4377_extract_msgid(struct bcm4377_data *bcm4377,
618 struct bcm4377_transfer_ring *ring,
619 u16 raw_msgid, u8 *msgid)
621 u8 generation = FIELD_GET(BCM4377_MSGID_GENERATION, raw_msgid);
622 *msgid = FIELD_GET(BCM4377_MSGID_ID, raw_msgid);
624 if (generation != ring->generation) {
627 "invalid message generation %d should be %d in entry for ring %d\n",
628 generation, ring->generation, ring->ring_id);
632 if (*msgid >= ring->n_entries) {
633 dev_warn(&bcm4377->pdev->dev,
634 "invalid message id in entry for ring %d: %d > %d\n",
635 ring->ring_id, *msgid, ring->n_entries);
642 static void bcm4377_handle_event(struct bcm4377_data *bcm4377,
643 struct bcm4377_transfer_ring *ring,
644 u16 raw_msgid, u8 entry_flags, u8 type,
645 void *payload, size_t len)
652 spin_lock_irqsave(&ring->lock, flags);
653 if (!ring->enabled) {
654 dev_warn(&bcm4377->pdev->dev,
655 "event for disabled transfer ring %d\n",
660 if (ring->d2h_buffers_only &&
661 entry_flags & BCM4377_XFER_RING_FLAG_PAYLOAD_MAPPED) {
662 if (bcm4377_extract_msgid(bcm4377, ring, raw_msgid, &msgid))
665 if (len > ring->mapped_payload_size) {
668 "invalid payload len in event for ring %d: %zu > %zu\n",
669 ring->ring_id, len, ring->mapped_payload_size);
673 payload = ring->payloads + msgid * ring->mapped_payload_size;
676 skb = bt_skb_alloc(len, GFP_ATOMIC);
680 memcpy(skb_put(skb, len), payload, len);
681 hci_skb_pkt_type(skb) = type;
682 hci_recv_frame(bcm4377->hdev, skb);
685 head = le16_to_cpu(bcm4377->ring_state->xfer_ring_head[ring->ring_id]);
686 head = (head + 1) % ring->n_entries;
687 bcm4377->ring_state->xfer_ring_head[ring->ring_id] = cpu_to_le16(head);
689 bcm4377_ring_doorbell(bcm4377, ring->doorbell, head);
691 spin_unlock_irqrestore(&ring->lock, flags);
694 static void bcm4377_handle_ack(struct bcm4377_data *bcm4377,
695 struct bcm4377_transfer_ring *ring,
701 spin_lock_irqsave(&ring->lock, flags);
703 if (bcm4377_extract_msgid(bcm4377, ring, raw_msgid, &msgid))
706 if (!test_bit(msgid, ring->msgids)) {
709 "invalid message id in ack for ring %d: %d is not used\n",
710 ring->ring_id, msgid);
714 if (ring->allow_wait && ring->events[msgid]) {
715 complete(ring->events[msgid]);
716 ring->events[msgid] = NULL;
719 bitmap_release_region(ring->msgids, msgid, ring->n_entries);
722 spin_unlock_irqrestore(&ring->lock, flags);
725 static void bcm4377_handle_completion(struct bcm4377_data *bcm4377,
726 struct bcm4377_completion_ring *ring,
729 struct bcm4377_completion_ring_entry *entry;
730 u16 msg_id, transfer_ring;
731 size_t entry_size, data_len;
734 if (pos >= ring->n_entries) {
735 dev_warn(&bcm4377->pdev->dev,
736 "invalid offset %d for completion ring %d\n", pos,
741 entry_size = sizeof(*entry) + ring->payload_size;
742 entry = ring->ring + pos * entry_size;
743 data = ring->ring + pos * entry_size + sizeof(*entry);
744 data_len = le32_to_cpu(entry->len);
745 msg_id = le16_to_cpu(entry->msg_id);
746 transfer_ring = le16_to_cpu(entry->ring_id);
748 if ((ring->transfer_rings & BIT(transfer_ring)) == 0) {
751 "invalid entry at offset %d for transfer ring %d in completion ring %d\n",
752 pos, transfer_ring, ring->ring_id);
756 dev_dbg(&bcm4377->pdev->dev,
757 "entry in completion ring %d for transfer ring %d with msg_id %d\n",
758 ring->ring_id, transfer_ring, msg_id);
760 switch (transfer_ring) {
761 case BCM4377_XFER_RING_CONTROL:
762 bcm4377_handle_ack(bcm4377, &bcm4377->control_h2d_ring, msg_id);
764 case BCM4377_XFER_RING_HCI_H2D:
765 bcm4377_handle_ack(bcm4377, &bcm4377->hci_h2d_ring, msg_id);
767 case BCM4377_XFER_RING_SCO_H2D:
768 bcm4377_handle_ack(bcm4377, &bcm4377->sco_h2d_ring, msg_id);
770 case BCM4377_XFER_RING_ACL_H2D:
771 bcm4377_handle_ack(bcm4377, &bcm4377->acl_h2d_ring, msg_id);
774 case BCM4377_XFER_RING_HCI_D2H:
775 bcm4377_handle_event(bcm4377, &bcm4377->hci_d2h_ring, msg_id,
776 entry->flags, HCI_EVENT_PKT, data,
779 case BCM4377_XFER_RING_SCO_D2H:
780 bcm4377_handle_event(bcm4377, &bcm4377->sco_d2h_ring, msg_id,
781 entry->flags, HCI_SCODATA_PKT, data,
784 case BCM4377_XFER_RING_ACL_D2H:
785 bcm4377_handle_event(bcm4377, &bcm4377->acl_d2h_ring, msg_id,
786 entry->flags, HCI_ACLDATA_PKT, data,
793 "entry in completion ring %d for unknown transfer ring %d with msg_id %d\n",
794 ring->ring_id, transfer_ring, msg_id);
798 static void bcm4377_poll_completion_ring(struct bcm4377_data *bcm4377,
799 struct bcm4377_completion_ring *ring)
802 __le16 *heads = bcm4377->ring_state->completion_ring_head;
803 __le16 *tails = bcm4377->ring_state->completion_ring_tail;
808 tail = le16_to_cpu(tails[ring->ring_id]);
809 dev_dbg(&bcm4377->pdev->dev,
810 "completion ring #%d: head: %d, tail: %d\n", ring->ring_id,
811 le16_to_cpu(heads[ring->ring_id]), tail);
813 while (tail != le16_to_cpu(READ_ONCE(heads[ring->ring_id]))) {
815 * ensure the CPU doesn't speculate through the comparison.
816 * otherwise it might already read the (empty) queue entry
817 * before the updated head has been loaded and checked.
821 bcm4377_handle_completion(bcm4377, ring, tail);
823 tail = (tail + 1) % ring->n_entries;
824 tails[ring->ring_id] = cpu_to_le16(tail);
828 static irqreturn_t bcm4377_irq(int irq, void *data)
830 struct bcm4377_data *bcm4377 = data;
831 u32 bootstage, rti_status;
833 bootstage = ioread32(bcm4377->bar2 + BCM4377_BAR2_BOOTSTAGE);
834 rti_status = ioread32(bcm4377->bar2 + BCM4377_BAR2_RTI_STATUS);
836 if (bootstage != bcm4377->bootstage ||
837 rti_status != bcm4377->rti_status) {
838 dev_dbg(&bcm4377->pdev->dev,
839 "bootstage = %d -> %d, rti state = %d -> %d\n",
840 bcm4377->bootstage, bootstage, bcm4377->rti_status,
842 complete(&bcm4377->event);
843 bcm4377->bootstage = bootstage;
844 bcm4377->rti_status = rti_status;
848 dev_err(&bcm4377->pdev->dev, "RTI status is %d\n", rti_status);
850 bcm4377_poll_completion_ring(bcm4377, &bcm4377->control_ack_ring);
851 bcm4377_poll_completion_ring(bcm4377, &bcm4377->hci_acl_event_ring);
852 bcm4377_poll_completion_ring(bcm4377, &bcm4377->hci_acl_ack_ring);
853 bcm4377_poll_completion_ring(bcm4377, &bcm4377->sco_ack_ring);
854 bcm4377_poll_completion_ring(bcm4377, &bcm4377->sco_event_ring);
859 static int bcm4377_enqueue(struct bcm4377_data *bcm4377,
860 struct bcm4377_transfer_ring *ring, void *data,
861 size_t len, bool wait)
864 struct bcm4377_xfer_ring_entry *entry;
867 u16 head, tail, new_head;
870 DECLARE_COMPLETION_ONSTACK(event);
872 if (len > ring->payload_size && len > ring->mapped_payload_size) {
875 "payload len %zu is too large for ring %d (max is %zu or %zu)\n",
876 len, ring->ring_id, ring->payload_size,
877 ring->mapped_payload_size);
880 if (wait && !ring->allow_wait)
885 spin_lock_irqsave(&ring->lock, flags);
887 head = le16_to_cpu(bcm4377->ring_state->xfer_ring_head[ring->ring_id]);
888 tail = le16_to_cpu(bcm4377->ring_state->xfer_ring_tail[ring->ring_id]);
890 new_head = (head + 1) % ring->n_entries;
892 if (new_head == tail) {
893 dev_warn(&bcm4377->pdev->dev,
894 "can't send message because ring %d is full\n",
900 msgid = bitmap_find_free_region(ring->msgids, ring->n_entries, 0);
902 dev_warn(&bcm4377->pdev->dev,
903 "can't find message id for ring %d\n", ring->ring_id);
908 raw_msgid = FIELD_PREP(BCM4377_MSGID_GENERATION, ring->generation);
909 raw_msgid |= FIELD_PREP(BCM4377_MSGID_ID, msgid);
911 offset = head * (sizeof(*entry) + ring->payload_size);
912 entry = ring->ring + offset;
914 memset(entry, 0, sizeof(*entry));
915 entry->id = cpu_to_le16(raw_msgid);
916 entry->len = cpu_to_le16(len);
918 if (len <= ring->payload_size) {
919 entry->flags = BCM4377_XFER_RING_FLAG_PAYLOAD_IN_FOOTER;
920 payload = ring->ring + offset + sizeof(*entry);
922 entry->flags = BCM4377_XFER_RING_FLAG_PAYLOAD_MAPPED;
923 entry->payload = cpu_to_le64(ring->payloads_dma +
924 msgid * ring->mapped_payload_size);
925 payload = ring->payloads + msgid * ring->mapped_payload_size;
928 memcpy(payload, data, len);
931 ring->events[msgid] = &event;
934 * The 4377 chips stop responding to any commands as soon as they
935 * have been idle for a while. Poking the sleep control register here
936 * makes them come alive again.
938 iowrite32(BCM4377_BAR0_SLEEP_CONTROL_AWAKE,
939 bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL);
941 dev_dbg(&bcm4377->pdev->dev,
942 "updating head for transfer queue #%d to %d\n", ring->ring_id,
944 bcm4377->ring_state->xfer_ring_head[ring->ring_id] =
945 cpu_to_le16(new_head);
948 bcm4377_ring_doorbell(bcm4377, ring->doorbell, new_head);
952 spin_unlock_irqrestore(&ring->lock, flags);
954 if (ret == 0 && wait) {
955 ret = wait_for_completion_interruptible_timeout(
956 &event, BCM4377_TIMEOUT);
962 spin_lock_irqsave(&ring->lock, flags);
963 ring->events[msgid] = NULL;
964 spin_unlock_irqrestore(&ring->lock, flags);
970 static int bcm4377_create_completion_ring(struct bcm4377_data *bcm4377,
971 struct bcm4377_completion_ring *ring)
973 struct bcm4377_create_completion_ring_msg msg;
977 dev_warn(&bcm4377->pdev->dev,
978 "completion ring %d already enabled\n", ring->ring_id);
982 memset(ring->ring, 0,
983 ring->n_entries * (sizeof(struct bcm4377_completion_ring_entry) +
984 ring->payload_size));
985 memset(&msg, 0, sizeof(msg));
986 msg.msg_type = BCM4377_CONTROL_MSG_CREATE_COMPLETION_RING;
987 msg.id = cpu_to_le16(ring->ring_id);
988 msg.id_again = cpu_to_le16(ring->ring_id);
989 msg.ring_iova = cpu_to_le64(ring->ring_dma);
990 msg.n_elements = cpu_to_le16(ring->n_entries);
991 msg.intmod_bytes = cpu_to_le32(0xffffffff);
992 msg.unk = cpu_to_le32(0xffffffff);
993 msg.intmod_delay = cpu_to_le16(ring->delay);
994 msg.footer_size = ring->payload_size / 4;
996 ret = bcm4377_enqueue(bcm4377, &bcm4377->control_h2d_ring, &msg,
999 ring->enabled = true;
1004 static int bcm4377_destroy_completion_ring(struct bcm4377_data *bcm4377,
1005 struct bcm4377_completion_ring *ring)
1007 struct bcm4377_destroy_completion_ring_msg msg;
1010 memset(&msg, 0, sizeof(msg));
1011 msg.msg_type = BCM4377_CONTROL_MSG_DESTROY_COMPLETION_RING;
1012 msg.ring_id = cpu_to_le16(ring->ring_id);
1014 ret = bcm4377_enqueue(bcm4377, &bcm4377->control_h2d_ring, &msg,
1017 dev_warn(&bcm4377->pdev->dev,
1018 "failed to destroy completion ring %d\n",
1021 ring->enabled = false;
1025 static int bcm4377_create_transfer_ring(struct bcm4377_data *bcm4377,
1026 struct bcm4377_transfer_ring *ring)
1028 struct bcm4377_create_transfer_ring_msg msg;
1031 unsigned long spinlock_flags;
1034 flags |= BCM4377_XFER_RING_FLAG_VIRTUAL;
1036 flags |= BCM4377_XFER_RING_FLAG_SYNC;
1038 spin_lock_irqsave(&ring->lock, spinlock_flags);
1039 memset(&msg, 0, sizeof(msg));
1040 msg.msg_type = BCM4377_CONTROL_MSG_CREATE_XFER_RING;
1041 msg.ring_id = cpu_to_le16(ring->ring_id);
1042 msg.ring_id_again = cpu_to_le16(ring->ring_id);
1043 msg.ring_iova = cpu_to_le64(ring->ring_dma);
1044 msg.n_elements = cpu_to_le16(ring->n_entries);
1045 msg.completion_ring_id = cpu_to_le16(ring->completion_ring);
1046 msg.doorbell = cpu_to_le16(ring->doorbell);
1047 msg.flags = cpu_to_le16(flags);
1048 msg.footer_size = ring->payload_size / 4;
1050 bcm4377->ring_state->xfer_ring_head[ring->ring_id] = 0;
1051 bcm4377->ring_state->xfer_ring_tail[ring->ring_id] = 0;
1053 spin_unlock_irqrestore(&ring->lock, spinlock_flags);
1055 ret = bcm4377_enqueue(bcm4377, &bcm4377->control_h2d_ring, &msg,
1058 spin_lock_irqsave(&ring->lock, spinlock_flags);
1060 if (ring->d2h_buffers_only) {
1061 for (i = 0; i < ring->n_entries; ++i) {
1062 struct bcm4377_xfer_ring_entry *entry =
1063 ring->ring + i * sizeof(*entry);
1064 u16 raw_msgid = FIELD_PREP(BCM4377_MSGID_GENERATION,
1066 raw_msgid |= FIELD_PREP(BCM4377_MSGID_ID, i);
1068 memset(entry, 0, sizeof(*entry));
1069 entry->id = cpu_to_le16(raw_msgid);
1070 entry->len = cpu_to_le16(ring->mapped_payload_size);
1071 entry->flags = BCM4377_XFER_RING_FLAG_PAYLOAD_MAPPED;
1073 cpu_to_le64(ring->payloads_dma +
1074 i * ring->mapped_payload_size);
1079 * send some messages if this is a device->host ring to allow the device
1080 * to reply by acknowledging them in the completion ring
1082 if (ring->virtual || ring->d2h_buffers_only) {
1083 bcm4377->ring_state->xfer_ring_head[ring->ring_id] =
1085 bcm4377_ring_doorbell(bcm4377, ring->doorbell, 0xf);
1088 ring->enabled = true;
1089 spin_unlock_irqrestore(&ring->lock, spinlock_flags);
1094 static int bcm4377_destroy_transfer_ring(struct bcm4377_data *bcm4377,
1095 struct bcm4377_transfer_ring *ring)
1097 struct bcm4377_destroy_transfer_ring_msg msg;
1100 memset(&msg, 0, sizeof(msg));
1101 msg.msg_type = BCM4377_CONTROL_MSG_DESTROY_XFER_RING;
1102 msg.ring_id = cpu_to_le16(ring->ring_id);
1104 ret = bcm4377_enqueue(bcm4377, &bcm4377->control_h2d_ring, &msg,
1107 dev_warn(&bcm4377->pdev->dev,
1108 "failed to destroy transfer ring %d\n", ring->ring_id);
1110 ring->enabled = false;
1114 static int __bcm4378_send_calibration_chunk(struct bcm4377_data *bcm4377,
1115 const void *data, size_t data_len,
1118 struct bcm4378_hci_send_calibration_cmd cmd;
1119 struct sk_buff *skb;
1121 if (data_len > sizeof(cmd.data))
1124 memset(&cmd, 0, sizeof(cmd));
1126 cmd.blocks_left = cpu_to_le16(blocks_left);
1127 memcpy(cmd.data, data, data_len);
1129 skb = __hci_cmd_sync(bcm4377->hdev, 0xfd97, sizeof(cmd), &cmd,
1132 return PTR_ERR(skb);
1138 static int __bcm4378_send_calibration(struct bcm4377_data *bcm4377,
1139 const void *data, size_t data_size)
1142 size_t i, left, transfer_len;
1144 DIV_ROUND_UP(data_size, (size_t)BCM4378_CALIBRATION_CHUNK_SIZE);
1147 dev_err(&bcm4377->pdev->dev,
1148 "no calibration data available.\n");
1152 for (i = 0, left = data_size; i < blocks; ++i, left -= transfer_len) {
1154 min_t(size_t, left, BCM4378_CALIBRATION_CHUNK_SIZE);
1156 ret = __bcm4378_send_calibration_chunk(
1157 bcm4377, data + i * BCM4378_CALIBRATION_CHUNK_SIZE,
1158 transfer_len, blocks - i - 1);
1160 dev_err(&bcm4377->pdev->dev,
1161 "send calibration chunk failed with %d\n", ret);
1169 static int bcm4378_send_calibration(struct bcm4377_data *bcm4377)
1171 if ((strcmp(bcm4377->stepping, "b1") == 0) ||
1172 strcmp(bcm4377->stepping, "b3") == 0)
1173 return __bcm4378_send_calibration(
1174 bcm4377, bcm4377->taurus_beamforming_cal_blob,
1175 bcm4377->taurus_beamforming_cal_size);
1177 return __bcm4378_send_calibration(bcm4377,
1178 bcm4377->taurus_cal_blob,
1179 bcm4377->taurus_cal_size);
1182 static int bcm4387_send_calibration(struct bcm4377_data *bcm4377)
1184 if (strcmp(bcm4377->stepping, "c2") == 0)
1185 return __bcm4378_send_calibration(
1186 bcm4377, bcm4377->taurus_beamforming_cal_blob,
1187 bcm4377->taurus_beamforming_cal_size);
1189 return __bcm4378_send_calibration(bcm4377,
1190 bcm4377->taurus_cal_blob,
1191 bcm4377->taurus_cal_size);
1194 static const struct firmware *bcm4377_request_blob(struct bcm4377_data *bcm4377,
1197 const struct firmware *fw;
1198 char name0[64], name1[64];
1201 snprintf(name0, sizeof(name0), "brcm/brcmbt%04x%s-%s-%s.%s",
1202 bcm4377->hw->id, bcm4377->stepping, bcm4377->board_type,
1203 bcm4377->vendor, suffix);
1204 snprintf(name1, sizeof(name1), "brcm/brcmbt%04x%s-%s.%s",
1205 bcm4377->hw->id, bcm4377->stepping, bcm4377->board_type,
1207 dev_dbg(&bcm4377->pdev->dev, "Trying to load firmware: '%s' or '%s'\n",
1210 ret = firmware_request_nowarn(&fw, name0, &bcm4377->pdev->dev);
1213 ret = firmware_request_nowarn(&fw, name1, &bcm4377->pdev->dev);
1217 dev_err(&bcm4377->pdev->dev,
1218 "Unable to load firmware; tried '%s' and '%s'\n", name0, name1);
1222 static int bcm4377_send_ptb(struct bcm4377_data *bcm4377,
1223 const struct firmware *fw)
1225 struct sk_buff *skb;
1227 skb = __hci_cmd_sync(bcm4377->hdev, 0xfd98, fw->size, fw->data,
1230 * This command seems to always fail on more recent firmware versions
1231 * (even in traces taken from the macOS driver). It's unclear why this
1232 * happens but because the PTB file contains calibration and/or
1233 * regulatory data and may be required on older firmware we still try to
1234 * send it here just in case and just ignore if it fails.
1241 static int bcm4378_send_ptb_chunk(struct bcm4377_data *bcm4377,
1242 const void *data, size_t data_len,
1245 struct bcm4378_hci_send_ptb_cmd cmd;
1246 struct sk_buff *skb;
1248 if (data_len > BCM4378_PTB_CHUNK_SIZE)
1251 memset(&cmd, 0, sizeof(cmd));
1252 cmd.blocks_left = cpu_to_le16(blocks_left);
1253 memcpy(cmd.data, data, data_len);
1255 skb = __hci_cmd_sync(bcm4377->hdev, 0xfe0d, sizeof(cmd), &cmd,
1258 return PTR_ERR(skb);
1264 static int bcm4378_send_ptb(struct bcm4377_data *bcm4377,
1265 const struct firmware *fw)
1267 size_t chunks = DIV_ROUND_UP(fw->size, (size_t)BCM4378_PTB_CHUNK_SIZE);
1268 size_t i, left, transfer_len;
1271 for (i = 0, left = fw->size; i < chunks; ++i, left -= transfer_len) {
1272 transfer_len = min_t(size_t, left, BCM4378_PTB_CHUNK_SIZE);
1274 dev_dbg(&bcm4377->pdev->dev, "sending ptb chunk %zu/%zu\n",
1276 ret = bcm4378_send_ptb_chunk(
1277 bcm4377, fw->data + i * BCM4378_PTB_CHUNK_SIZE,
1278 transfer_len, chunks - i - 1);
1280 dev_err(&bcm4377->pdev->dev,
1281 "sending ptb chunk %zu failed (%d)", i, ret);
1289 static int bcm4377_hci_open(struct hci_dev *hdev)
1291 struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev);
1294 dev_dbg(&bcm4377->pdev->dev, "creating rings\n");
1296 ret = bcm4377_create_completion_ring(bcm4377,
1297 &bcm4377->hci_acl_ack_ring);
1300 ret = bcm4377_create_completion_ring(bcm4377,
1301 &bcm4377->hci_acl_event_ring);
1303 goto destroy_hci_acl_ack;
1304 ret = bcm4377_create_completion_ring(bcm4377, &bcm4377->sco_ack_ring);
1306 goto destroy_hci_acl_event;
1307 ret = bcm4377_create_completion_ring(bcm4377, &bcm4377->sco_event_ring);
1309 goto destroy_sco_ack;
1310 dev_dbg(&bcm4377->pdev->dev,
1311 "all completion rings successfully created!\n");
1313 ret = bcm4377_create_transfer_ring(bcm4377, &bcm4377->hci_h2d_ring);
1315 goto destroy_sco_event;
1316 ret = bcm4377_create_transfer_ring(bcm4377, &bcm4377->hci_d2h_ring);
1318 goto destroy_hci_h2d;
1319 ret = bcm4377_create_transfer_ring(bcm4377, &bcm4377->sco_h2d_ring);
1321 goto destroy_hci_d2h;
1322 ret = bcm4377_create_transfer_ring(bcm4377, &bcm4377->sco_d2h_ring);
1324 goto destroy_sco_h2d;
1325 ret = bcm4377_create_transfer_ring(bcm4377, &bcm4377->acl_h2d_ring);
1327 goto destroy_sco_d2h;
1328 ret = bcm4377_create_transfer_ring(bcm4377, &bcm4377->acl_d2h_ring);
1330 goto destroy_acl_h2d;
1331 dev_dbg(&bcm4377->pdev->dev,
1332 "all transfer rings successfully created!\n");
1337 bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->acl_h2d_ring);
1339 bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->sco_d2h_ring);
1341 bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->sco_h2d_ring);
1343 bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->hci_h2d_ring);
1345 bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->hci_d2h_ring);
1347 bcm4377_destroy_completion_ring(bcm4377, &bcm4377->sco_event_ring);
1349 bcm4377_destroy_completion_ring(bcm4377, &bcm4377->sco_ack_ring);
1350 destroy_hci_acl_event:
1351 bcm4377_destroy_completion_ring(bcm4377, &bcm4377->hci_acl_event_ring);
1352 destroy_hci_acl_ack:
1353 bcm4377_destroy_completion_ring(bcm4377, &bcm4377->hci_acl_ack_ring);
1355 dev_err(&bcm4377->pdev->dev, "Creating rings failed with %d\n", ret);
1359 static int bcm4377_hci_close(struct hci_dev *hdev)
1361 struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev);
1363 dev_dbg(&bcm4377->pdev->dev, "destroying rings in hci_close\n");
1365 bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->acl_d2h_ring);
1366 bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->acl_h2d_ring);
1367 bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->sco_d2h_ring);
1368 bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->sco_h2d_ring);
1369 bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->hci_d2h_ring);
1370 bcm4377_destroy_transfer_ring(bcm4377, &bcm4377->hci_h2d_ring);
1372 bcm4377_destroy_completion_ring(bcm4377, &bcm4377->sco_event_ring);
1373 bcm4377_destroy_completion_ring(bcm4377, &bcm4377->sco_ack_ring);
1374 bcm4377_destroy_completion_ring(bcm4377, &bcm4377->hci_acl_event_ring);
1375 bcm4377_destroy_completion_ring(bcm4377, &bcm4377->hci_acl_ack_ring);
1380 static bool bcm4377_is_valid_bdaddr(struct bcm4377_data *bcm4377,
1383 if (addr->b[0] != 0x93)
1385 if (addr->b[1] != 0x76)
1387 if (addr->b[2] != 0x00)
1389 if (addr->b[4] != (bcm4377->hw->id & 0xff))
1391 if (addr->b[5] != (bcm4377->hw->id >> 8))
1396 static int bcm4377_check_bdaddr(struct bcm4377_data *bcm4377)
1398 struct hci_rp_read_bd_addr *bda;
1399 struct sk_buff *skb;
1401 skb = __hci_cmd_sync(bcm4377->hdev, HCI_OP_READ_BD_ADDR, 0, NULL,
1404 int err = PTR_ERR(skb);
1406 dev_err(&bcm4377->pdev->dev, "HCI_OP_READ_BD_ADDR failed (%d)",
1411 if (skb->len != sizeof(*bda)) {
1412 dev_err(&bcm4377->pdev->dev,
1413 "HCI_OP_READ_BD_ADDR reply length invalid");
1418 bda = (struct hci_rp_read_bd_addr *)skb->data;
1419 if (!bcm4377_is_valid_bdaddr(bcm4377, &bda->bdaddr))
1420 set_bit(HCI_QUIRK_INVALID_BDADDR, &bcm4377->hdev->quirks);
1426 static int bcm4377_hci_setup(struct hci_dev *hdev)
1428 struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev);
1429 const struct firmware *fw;
1432 if (bcm4377->hw->send_calibration) {
1433 ret = bcm4377->hw->send_calibration(bcm4377);
1438 fw = bcm4377_request_blob(bcm4377, "ptb");
1440 dev_err(&bcm4377->pdev->dev, "failed to load PTB data");
1444 ret = bcm4377->hw->send_ptb(bcm4377, fw);
1445 release_firmware(fw);
1449 return bcm4377_check_bdaddr(bcm4377);
1452 static int bcm4377_hci_send_frame(struct hci_dev *hdev, struct sk_buff *skb)
1454 struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev);
1455 struct bcm4377_transfer_ring *ring;
1458 switch (hci_skb_pkt_type(skb)) {
1459 case HCI_COMMAND_PKT:
1460 hdev->stat.cmd_tx++;
1461 ring = &bcm4377->hci_h2d_ring;
1464 case HCI_ACLDATA_PKT:
1465 hdev->stat.acl_tx++;
1466 ring = &bcm4377->acl_h2d_ring;
1469 case HCI_SCODATA_PKT:
1470 hdev->stat.sco_tx++;
1471 ring = &bcm4377->sco_h2d_ring;
1478 ret = bcm4377_enqueue(bcm4377, ring, skb->data, skb->len, false);
1480 hdev->stat.err_tx++;
1484 hdev->stat.byte_tx += skb->len;
1489 static int bcm4377_hci_set_bdaddr(struct hci_dev *hdev, const bdaddr_t *bdaddr)
1491 struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev);
1492 struct sk_buff *skb;
1495 skb = __hci_cmd_sync(hdev, 0xfc01, 6, bdaddr, HCI_INIT_TIMEOUT);
1498 dev_err(&bcm4377->pdev->dev,
1499 "Change address command failed (%d)", err);
1507 static int bcm4377_alloc_transfer_ring(struct bcm4377_data *bcm4377,
1508 struct bcm4377_transfer_ring *ring)
1512 spin_lock_init(&ring->lock);
1513 ring->payload_size = ALIGN(ring->payload_size, 4);
1514 ring->mapped_payload_size = ALIGN(ring->mapped_payload_size, 4);
1516 if (ring->payload_size > BCM4377_XFER_RING_MAX_INPLACE_PAYLOAD_SIZE)
1518 if (ring->n_entries > BCM4377_MAX_RING_SIZE)
1520 if (ring->virtual && ring->allow_wait)
1523 if (ring->d2h_buffers_only) {
1526 if (ring->payload_size)
1528 if (!ring->mapped_payload_size)
1535 ring->payload_size + sizeof(struct bcm4377_xfer_ring_entry);
1536 ring->ring = dmam_alloc_coherent(&bcm4377->pdev->dev,
1537 ring->n_entries * entry_size,
1538 &ring->ring_dma, GFP_KERNEL);
1542 if (ring->allow_wait) {
1543 ring->events = devm_kcalloc(&bcm4377->pdev->dev,
1545 sizeof(*ring->events), GFP_KERNEL);
1550 if (ring->mapped_payload_size) {
1551 ring->payloads = dmam_alloc_coherent(
1552 &bcm4377->pdev->dev,
1553 ring->n_entries * ring->mapped_payload_size,
1554 &ring->payloads_dma, GFP_KERNEL);
1555 if (!ring->payloads)
1562 static int bcm4377_alloc_completion_ring(struct bcm4377_data *bcm4377,
1563 struct bcm4377_completion_ring *ring)
1567 ring->payload_size = ALIGN(ring->payload_size, 4);
1568 if (ring->payload_size > BCM4377_XFER_RING_MAX_INPLACE_PAYLOAD_SIZE)
1570 if (ring->n_entries > BCM4377_MAX_RING_SIZE)
1573 entry_size = ring->payload_size +
1574 sizeof(struct bcm4377_completion_ring_entry);
1576 ring->ring = dmam_alloc_coherent(&bcm4377->pdev->dev,
1577 ring->n_entries * entry_size,
1578 &ring->ring_dma, GFP_KERNEL);
1584 static int bcm4377_init_context(struct bcm4377_data *bcm4377)
1586 struct device *dev = &bcm4377->pdev->dev;
1587 dma_addr_t peripheral_info_dma;
1589 bcm4377->ctx = dmam_alloc_coherent(dev, sizeof(*bcm4377->ctx),
1590 &bcm4377->ctx_dma, GFP_KERNEL);
1593 memset(bcm4377->ctx, 0, sizeof(*bcm4377->ctx));
1595 bcm4377->ring_state =
1596 dmam_alloc_coherent(dev, sizeof(*bcm4377->ring_state),
1597 &bcm4377->ring_state_dma, GFP_KERNEL);
1598 if (!bcm4377->ring_state)
1600 memset(bcm4377->ring_state, 0, sizeof(*bcm4377->ring_state));
1602 bcm4377->ctx->version = cpu_to_le16(1);
1603 bcm4377->ctx->size = cpu_to_le16(sizeof(*bcm4377->ctx));
1604 bcm4377->ctx->enabled_caps = cpu_to_le32(2);
1607 * The BT device will write 0x20 bytes of data to this buffer but
1608 * the exact contents are unknown. It only needs to exist for BT
1609 * to work such that we can just allocate and then ignore it.
1611 if (!dmam_alloc_coherent(&bcm4377->pdev->dev, 0x20,
1612 &peripheral_info_dma, GFP_KERNEL))
1614 bcm4377->ctx->peripheral_info_addr = cpu_to_le64(peripheral_info_dma);
1616 bcm4377->ctx->xfer_ring_heads_addr = cpu_to_le64(
1617 bcm4377->ring_state_dma +
1618 offsetof(struct bcm4377_ring_state, xfer_ring_head));
1619 bcm4377->ctx->xfer_ring_tails_addr = cpu_to_le64(
1620 bcm4377->ring_state_dma +
1621 offsetof(struct bcm4377_ring_state, xfer_ring_tail));
1622 bcm4377->ctx->completion_ring_heads_addr = cpu_to_le64(
1623 bcm4377->ring_state_dma +
1624 offsetof(struct bcm4377_ring_state, completion_ring_head));
1625 bcm4377->ctx->completion_ring_tails_addr = cpu_to_le64(
1626 bcm4377->ring_state_dma +
1627 offsetof(struct bcm4377_ring_state, completion_ring_tail));
1629 bcm4377->ctx->n_completion_rings =
1630 cpu_to_le16(BCM4377_N_COMPLETION_RINGS);
1631 bcm4377->ctx->n_xfer_rings = cpu_to_le16(BCM4377_N_TRANSFER_RINGS);
1633 bcm4377->ctx->control_completion_ring_addr =
1634 cpu_to_le64(bcm4377->control_ack_ring.ring_dma);
1635 bcm4377->ctx->control_completion_ring_n_entries =
1636 cpu_to_le16(bcm4377->control_ack_ring.n_entries);
1637 bcm4377->ctx->control_completion_ring_doorbell = cpu_to_le16(0xffff);
1638 bcm4377->ctx->control_completion_ring_msi = 0;
1639 bcm4377->ctx->control_completion_ring_header_size = 0;
1640 bcm4377->ctx->control_completion_ring_footer_size = 0;
1642 bcm4377->ctx->control_xfer_ring_addr =
1643 cpu_to_le64(bcm4377->control_h2d_ring.ring_dma);
1644 bcm4377->ctx->control_xfer_ring_n_entries =
1645 cpu_to_le16(bcm4377->control_h2d_ring.n_entries);
1646 bcm4377->ctx->control_xfer_ring_doorbell =
1647 cpu_to_le16(bcm4377->control_h2d_ring.doorbell);
1648 bcm4377->ctx->control_xfer_ring_msi = 0;
1649 bcm4377->ctx->control_xfer_ring_header_size = 0;
1650 bcm4377->ctx->control_xfer_ring_footer_size =
1651 bcm4377->control_h2d_ring.payload_size / 4;
1653 dev_dbg(&bcm4377->pdev->dev, "context initialized at IOVA %pad",
1659 static int bcm4377_prepare_rings(struct bcm4377_data *bcm4377)
1664 * Even though many of these settings appear to be configurable
1665 * when sending the "create ring" messages most of these are
1666 * actually hardcoded in some (and quite possibly all) firmware versions
1667 * and changing them on the host has no effect.
1668 * Specifically, this applies to at least the doorbells, the transfer
1669 * and completion ring ids and their mapping (e.g. both HCI and ACL
1670 * entries will always be queued in completion rings 1 and 2 no matter
1671 * what we configure here).
1673 bcm4377->control_ack_ring.ring_id = BCM4377_ACK_RING_CONTROL;
1674 bcm4377->control_ack_ring.n_entries = 32;
1675 bcm4377->control_ack_ring.transfer_rings =
1676 BIT(BCM4377_XFER_RING_CONTROL);
1678 bcm4377->hci_acl_ack_ring.ring_id = BCM4377_ACK_RING_HCI_ACL;
1679 bcm4377->hci_acl_ack_ring.n_entries = 2 * BCM4377_RING_N_ENTRIES;
1680 bcm4377->hci_acl_ack_ring.transfer_rings =
1681 BIT(BCM4377_XFER_RING_HCI_H2D) | BIT(BCM4377_XFER_RING_ACL_H2D);
1682 bcm4377->hci_acl_ack_ring.delay = 1000;
1685 * A payload size of MAX_EVENT_PAYLOAD_SIZE is enough here since large
1686 * ACL packets will be transmitted inside buffers mapped via
1687 * acl_d2h_ring anyway.
1689 bcm4377->hci_acl_event_ring.ring_id = BCM4377_EVENT_RING_HCI_ACL;
1690 bcm4377->hci_acl_event_ring.payload_size = MAX_EVENT_PAYLOAD_SIZE;
1691 bcm4377->hci_acl_event_ring.n_entries = 2 * BCM4377_RING_N_ENTRIES;
1692 bcm4377->hci_acl_event_ring.transfer_rings =
1693 BIT(BCM4377_XFER_RING_HCI_D2H) | BIT(BCM4377_XFER_RING_ACL_D2H);
1694 bcm4377->hci_acl_event_ring.delay = 1000;
1696 bcm4377->sco_ack_ring.ring_id = BCM4377_ACK_RING_SCO;
1697 bcm4377->sco_ack_ring.n_entries = BCM4377_RING_N_ENTRIES;
1698 bcm4377->sco_ack_ring.transfer_rings = BIT(BCM4377_XFER_RING_SCO_H2D);
1700 bcm4377->sco_event_ring.ring_id = BCM4377_EVENT_RING_SCO;
1701 bcm4377->sco_event_ring.payload_size = MAX_SCO_PAYLOAD_SIZE;
1702 bcm4377->sco_event_ring.n_entries = BCM4377_RING_N_ENTRIES;
1703 bcm4377->sco_event_ring.transfer_rings = BIT(BCM4377_XFER_RING_SCO_D2H);
1705 bcm4377->control_h2d_ring.ring_id = BCM4377_XFER_RING_CONTROL;
1706 bcm4377->control_h2d_ring.doorbell = BCM4377_DOORBELL_CONTROL;
1707 bcm4377->control_h2d_ring.payload_size = BCM4377_CONTROL_MSG_SIZE;
1708 bcm4377->control_h2d_ring.completion_ring = BCM4377_ACK_RING_CONTROL;
1709 bcm4377->control_h2d_ring.allow_wait = true;
1710 bcm4377->control_h2d_ring.n_entries = BCM4377_RING_N_ENTRIES;
1712 bcm4377->hci_h2d_ring.ring_id = BCM4377_XFER_RING_HCI_H2D;
1713 bcm4377->hci_h2d_ring.doorbell = BCM4377_DOORBELL_HCI_H2D;
1714 bcm4377->hci_h2d_ring.payload_size = MAX_EVENT_PAYLOAD_SIZE;
1715 bcm4377->hci_h2d_ring.completion_ring = BCM4377_ACK_RING_HCI_ACL;
1716 bcm4377->hci_h2d_ring.n_entries = BCM4377_RING_N_ENTRIES;
1718 bcm4377->hci_d2h_ring.ring_id = BCM4377_XFER_RING_HCI_D2H;
1719 bcm4377->hci_d2h_ring.doorbell = BCM4377_DOORBELL_HCI_D2H;
1720 bcm4377->hci_d2h_ring.completion_ring = BCM4377_EVENT_RING_HCI_ACL;
1721 bcm4377->hci_d2h_ring.virtual = true;
1722 bcm4377->hci_d2h_ring.n_entries = BCM4377_RING_N_ENTRIES;
1724 bcm4377->sco_h2d_ring.ring_id = BCM4377_XFER_RING_SCO_H2D;
1725 bcm4377->sco_h2d_ring.doorbell = BCM4377_DOORBELL_SCO;
1726 bcm4377->sco_h2d_ring.payload_size = MAX_SCO_PAYLOAD_SIZE;
1727 bcm4377->sco_h2d_ring.completion_ring = BCM4377_ACK_RING_SCO;
1728 bcm4377->sco_h2d_ring.sync = true;
1729 bcm4377->sco_h2d_ring.n_entries = BCM4377_RING_N_ENTRIES;
1731 bcm4377->sco_d2h_ring.ring_id = BCM4377_XFER_RING_SCO_D2H;
1732 bcm4377->sco_d2h_ring.doorbell = BCM4377_DOORBELL_SCO;
1733 bcm4377->sco_d2h_ring.completion_ring = BCM4377_EVENT_RING_SCO;
1734 bcm4377->sco_d2h_ring.virtual = true;
1735 bcm4377->sco_d2h_ring.sync = true;
1736 bcm4377->sco_d2h_ring.n_entries = BCM4377_RING_N_ENTRIES;
1739 * This ring has to use mapped_payload_size because the largest ACL
1740 * packet doesn't fit inside the largest possible footer
1742 bcm4377->acl_h2d_ring.ring_id = BCM4377_XFER_RING_ACL_H2D;
1743 bcm4377->acl_h2d_ring.doorbell = BCM4377_DOORBELL_ACL_H2D;
1744 bcm4377->acl_h2d_ring.mapped_payload_size = MAX_ACL_PAYLOAD_SIZE;
1745 bcm4377->acl_h2d_ring.completion_ring = BCM4377_ACK_RING_HCI_ACL;
1746 bcm4377->acl_h2d_ring.n_entries = BCM4377_RING_N_ENTRIES;
1749 * This ring only contains empty buffers to be used by incoming
1750 * ACL packets that do not fit inside the footer of hci_acl_event_ring
1752 bcm4377->acl_d2h_ring.ring_id = BCM4377_XFER_RING_ACL_D2H;
1753 bcm4377->acl_d2h_ring.doorbell = BCM4377_DOORBELL_ACL_D2H;
1754 bcm4377->acl_d2h_ring.completion_ring = BCM4377_EVENT_RING_HCI_ACL;
1755 bcm4377->acl_d2h_ring.d2h_buffers_only = true;
1756 bcm4377->acl_d2h_ring.mapped_payload_size = MAX_ACL_PAYLOAD_SIZE;
1757 bcm4377->acl_d2h_ring.n_entries = BCM4377_RING_N_ENTRIES;
1760 * no need for any cleanup since this is only called from _probe
1761 * and only devres-managed allocations are used
1763 ret = bcm4377_alloc_transfer_ring(bcm4377, &bcm4377->control_h2d_ring);
1766 ret = bcm4377_alloc_transfer_ring(bcm4377, &bcm4377->hci_h2d_ring);
1769 ret = bcm4377_alloc_transfer_ring(bcm4377, &bcm4377->hci_d2h_ring);
1772 ret = bcm4377_alloc_transfer_ring(bcm4377, &bcm4377->sco_h2d_ring);
1775 ret = bcm4377_alloc_transfer_ring(bcm4377, &bcm4377->sco_d2h_ring);
1778 ret = bcm4377_alloc_transfer_ring(bcm4377, &bcm4377->acl_h2d_ring);
1781 ret = bcm4377_alloc_transfer_ring(bcm4377, &bcm4377->acl_d2h_ring);
1785 ret = bcm4377_alloc_completion_ring(bcm4377,
1786 &bcm4377->control_ack_ring);
1789 ret = bcm4377_alloc_completion_ring(bcm4377,
1790 &bcm4377->hci_acl_ack_ring);
1793 ret = bcm4377_alloc_completion_ring(bcm4377,
1794 &bcm4377->hci_acl_event_ring);
1797 ret = bcm4377_alloc_completion_ring(bcm4377, &bcm4377->sco_ack_ring);
1800 ret = bcm4377_alloc_completion_ring(bcm4377, &bcm4377->sco_event_ring);
1804 dev_dbg(&bcm4377->pdev->dev, "all rings allocated and prepared\n");
1809 static int bcm4377_boot(struct bcm4377_data *bcm4377)
1811 const struct firmware *fw;
1815 u32 bootstage, rti_status;
1817 bootstage = ioread32(bcm4377->bar2 + BCM4377_BAR2_BOOTSTAGE);
1818 rti_status = ioread32(bcm4377->bar2 + BCM4377_BAR2_RTI_STATUS);
1820 if (bootstage != 0) {
1821 dev_err(&bcm4377->pdev->dev, "bootstage is %d and not 0\n",
1826 if (rti_status != 0) {
1827 dev_err(&bcm4377->pdev->dev, "RTI status is %d and not 0\n",
1832 fw = bcm4377_request_blob(bcm4377, "bin");
1834 dev_err(&bcm4377->pdev->dev, "Failed to load firmware\n");
1838 bfr = dma_alloc_coherent(&bcm4377->pdev->dev, fw->size, &fw_dma,
1842 goto out_release_fw;
1845 memcpy(bfr, fw->data, fw->size);
1847 iowrite32(0, bcm4377->bar0 + BCM4377_BAR0_HOST_WINDOW_LO);
1848 iowrite32(0, bcm4377->bar0 + BCM4377_BAR0_HOST_WINDOW_HI);
1849 iowrite32(BCM4377_DMA_MASK,
1850 bcm4377->bar0 + BCM4377_BAR0_HOST_WINDOW_SIZE);
1852 iowrite32(lower_32_bits(fw_dma), bcm4377->bar2 + BCM4377_BAR2_FW_LO);
1853 iowrite32(upper_32_bits(fw_dma), bcm4377->bar2 + BCM4377_BAR2_FW_HI);
1854 iowrite32(fw->size, bcm4377->bar2 + BCM4377_BAR2_FW_SIZE);
1855 iowrite32(0, bcm4377->bar0 + BCM4377_BAR0_FW_DOORBELL);
1857 dev_dbg(&bcm4377->pdev->dev, "waiting for firmware to boot\n");
1859 ret = wait_for_completion_interruptible_timeout(&bcm4377->event,
1864 } else if (ret < 0) {
1868 if (bcm4377->bootstage != 2) {
1869 dev_err(&bcm4377->pdev->dev, "boostage %d != 2\n",
1870 bcm4377->bootstage);
1875 dev_dbg(&bcm4377->pdev->dev, "firmware has booted (stage = %x)\n",
1876 bcm4377->bootstage);
1880 dma_free_coherent(&bcm4377->pdev->dev, fw->size, bfr, fw_dma);
1882 release_firmware(fw);
1886 static int bcm4377_setup_rti(struct bcm4377_data *bcm4377)
1890 dev_dbg(&bcm4377->pdev->dev, "starting RTI\n");
1891 iowrite32(1, bcm4377->bar0 + BCM4377_BAR0_RTI_CONTROL);
1893 ret = wait_for_completion_interruptible_timeout(&bcm4377->event,
1896 dev_err(&bcm4377->pdev->dev,
1897 "timed out while waiting for RTI to transition to state 1");
1899 } else if (ret < 0) {
1903 if (bcm4377->rti_status != 1) {
1904 dev_err(&bcm4377->pdev->dev, "RTI did not ack state 1 (%d)\n",
1905 bcm4377->rti_status);
1908 dev_dbg(&bcm4377->pdev->dev, "RTI is in state 1\n");
1910 /* allow access to the entire IOVA space again */
1911 iowrite32(0, bcm4377->bar2 + BCM4377_BAR2_RTI_WINDOW_LO);
1912 iowrite32(0, bcm4377->bar2 + BCM4377_BAR2_RTI_WINDOW_HI);
1913 iowrite32(BCM4377_DMA_MASK,
1914 bcm4377->bar2 + BCM4377_BAR2_RTI_WINDOW_SIZE);
1916 /* setup "Converged IPC" context */
1917 iowrite32(lower_32_bits(bcm4377->ctx_dma),
1918 bcm4377->bar2 + BCM4377_BAR2_CONTEXT_ADDR_LO);
1919 iowrite32(upper_32_bits(bcm4377->ctx_dma),
1920 bcm4377->bar2 + BCM4377_BAR2_CONTEXT_ADDR_HI);
1921 iowrite32(2, bcm4377->bar0 + BCM4377_BAR0_RTI_CONTROL);
1923 ret = wait_for_completion_interruptible_timeout(&bcm4377->event,
1926 dev_err(&bcm4377->pdev->dev,
1927 "timed out while waiting for RTI to transition to state 2");
1929 } else if (ret < 0) {
1933 if (bcm4377->rti_status != 2) {
1934 dev_err(&bcm4377->pdev->dev, "RTI did not ack state 2 (%d)\n",
1935 bcm4377->rti_status);
1939 dev_dbg(&bcm4377->pdev->dev,
1940 "RTI is in state 2; control ring is ready\n");
1941 bcm4377->control_ack_ring.enabled = true;
1946 static int bcm4377_parse_otp_board_params(struct bcm4377_data *bcm4377,
1947 char tag, const char *val, size_t len)
1951 if (len >= sizeof(bcm4377->vendor))
1954 strscpy(bcm4377->vendor, val, len + 1);
1958 static int bcm4377_parse_otp_chip_params(struct bcm4377_data *bcm4377, char tag,
1959 const char *val, size_t len)
1965 if (len >= sizeof(bcm4377->stepping))
1969 bcm4377->stepping[idx] = tolower(val[idx]);
1970 if (val[idx] == '\0')
1977 bcm4377->stepping[idx] = '\0';
1981 static int bcm4377_parse_otp_str(struct bcm4377_data *bcm4377, const u8 *str,
1982 enum bcm4377_otp_params_type type)
1987 p = skip_spaces(str);
1993 if (*p++ != '=') /* implicit NUL check */
1996 /* *p might be NUL here, if so end == p and len == 0 */
1997 end = strchrnul(p, ' ');
2000 /* leave 1 byte for NUL in destination string */
2001 if (len > (BCM4377_OTP_MAX_PARAM_LEN - 1))
2005 case BCM4377_OTP_BOARD_PARAMS:
2006 ret = bcm4377_parse_otp_board_params(bcm4377, tag, p,
2009 case BCM4377_OTP_CHIP_PARAMS:
2010 ret = bcm4377_parse_otp_chip_params(bcm4377, tag, p,
2021 /* Skip to next arg, if any */
2022 p = skip_spaces(end);
2028 static int bcm4377_parse_otp_sys_vendor(struct bcm4377_data *bcm4377, u8 *otp,
2032 const char *chip_params;
2033 const char *board_params;
2036 /* 4-byte header and two empty strings */
2040 if (get_unaligned_le32(otp) != BCM4377_OTP_VENDOR_HDR)
2043 chip_params = &otp[idx];
2045 /* Skip first string, including terminator */
2046 idx += strnlen(chip_params, size - idx) + 1;
2050 board_params = &otp[idx];
2052 /* Skip to terminator of second string */
2053 idx += strnlen(board_params, size - idx);
2057 /* At this point both strings are guaranteed NUL-terminated */
2058 dev_dbg(&bcm4377->pdev->dev,
2059 "OTP: chip_params='%s' board_params='%s'\n", chip_params,
2062 ret = bcm4377_parse_otp_str(bcm4377, chip_params,
2063 BCM4377_OTP_CHIP_PARAMS);
2067 ret = bcm4377_parse_otp_str(bcm4377, board_params,
2068 BCM4377_OTP_BOARD_PARAMS);
2072 if (!bcm4377->stepping[0] || !bcm4377->vendor[0])
2075 dev_dbg(&bcm4377->pdev->dev, "OTP: stepping=%s, vendor=%s\n",
2076 bcm4377->stepping, bcm4377->vendor);
2080 static int bcm4377_parse_otp(struct bcm4377_data *bcm4377)
2086 otp = kzalloc(BCM4377_OTP_SIZE, GFP_KERNEL);
2090 for (i = 0; i < BCM4377_OTP_SIZE; ++i)
2091 otp[i] = ioread8(bcm4377->bar0 + bcm4377->hw->otp_offset + i);
2094 while (i < (BCM4377_OTP_SIZE - 1)) {
2096 u8 length = otp[i + 1];
2101 if ((i + 2 + length) > BCM4377_OTP_SIZE)
2105 case BCM4377_OTP_SYS_VENDOR:
2106 dev_dbg(&bcm4377->pdev->dev,
2107 "OTP @ 0x%x (%d): SYS_VENDOR", i, length);
2108 ret = bcm4377_parse_otp_sys_vendor(bcm4377, &otp[i + 2],
2111 case BCM4377_OTP_CIS:
2112 dev_dbg(&bcm4377->pdev->dev, "OTP @ 0x%x (%d): CIS", i,
2116 dev_dbg(&bcm4377->pdev->dev, "OTP @ 0x%x (%d): unknown",
2128 static int bcm4377_init_cfg(struct bcm4377_data *bcm4377)
2133 ret = pci_write_config_dword(bcm4377->pdev,
2134 BCM4377_PCIECFG_BAR0_WINDOW1,
2135 bcm4377->hw->bar0_window1);
2139 ret = pci_write_config_dword(bcm4377->pdev,
2140 BCM4377_PCIECFG_BAR0_WINDOW2,
2141 bcm4377->hw->bar0_window2);
2145 ret = pci_write_config_dword(
2146 bcm4377->pdev, BCM4377_PCIECFG_BAR0_CORE2_WINDOW1,
2147 BCM4377_PCIECFG_BAR0_CORE2_WINDOW1_DEFAULT);
2151 if (bcm4377->hw->has_bar0_core2_window2) {
2152 ret = pci_write_config_dword(bcm4377->pdev,
2153 BCM4377_PCIECFG_BAR0_CORE2_WINDOW2,
2154 bcm4377->hw->bar0_core2_window2);
2159 ret = pci_write_config_dword(bcm4377->pdev, BCM4377_PCIECFG_BAR2_WINDOW,
2160 BCM4377_PCIECFG_BAR2_WINDOW_DEFAULT);
2164 ret = pci_read_config_dword(bcm4377->pdev,
2165 BCM4377_PCIECFG_SUBSYSTEM_CTRL, &ctrl);
2169 if (bcm4377->hw->clear_pciecfg_subsystem_ctrl_bit19)
2173 return pci_write_config_dword(bcm4377->pdev,
2174 BCM4377_PCIECFG_SUBSYSTEM_CTRL, ctrl);
2177 static int bcm4377_probe_dmi(struct bcm4377_data *bcm4377)
2179 const struct dmi_system_id *board_type_dmi_id;
2181 board_type_dmi_id = dmi_first_match(bcm4377_dmi_board_table);
2182 if (board_type_dmi_id && board_type_dmi_id->driver_data) {
2183 bcm4377->board_type = board_type_dmi_id->driver_data;
2184 dev_dbg(&bcm4377->pdev->dev,
2185 "found board type via DMI match: %s\n",
2186 bcm4377->board_type);
2192 static int bcm4377_probe_of(struct bcm4377_data *bcm4377)
2194 struct device_node *np = bcm4377->pdev->dev.of_node;
2200 ret = of_property_read_string(np, "brcm,board-type",
2201 &bcm4377->board_type);
2203 dev_err(&bcm4377->pdev->dev, "no brcm,board-type property\n");
2207 bcm4377->taurus_beamforming_cal_blob =
2208 of_get_property(np, "brcm,taurus-bf-cal-blob",
2209 &bcm4377->taurus_beamforming_cal_size);
2210 if (!bcm4377->taurus_beamforming_cal_blob) {
2211 dev_err(&bcm4377->pdev->dev,
2212 "no brcm,taurus-bf-cal-blob property\n");
2215 bcm4377->taurus_cal_blob = of_get_property(np, "brcm,taurus-cal-blob",
2216 &bcm4377->taurus_cal_size);
2217 if (!bcm4377->taurus_cal_blob) {
2218 dev_err(&bcm4377->pdev->dev,
2219 "no brcm,taurus-cal-blob property\n");
2226 static void bcm4377_disable_aspm(struct bcm4377_data *bcm4377)
2228 pci_disable_link_state(bcm4377->pdev,
2229 PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1);
2232 * pci_disable_link_state can fail if either CONFIG_PCIEASPM is disabled
2233 * or if the BIOS hasn't handed over control to us. We must *always*
2234 * disable ASPM for this device due to hardware errata though.
2236 pcie_capability_clear_word(bcm4377->pdev, PCI_EXP_LNKCTL,
2237 PCI_EXP_LNKCTL_ASPMC);
2240 static void bcm4377_pci_free_irq_vectors(void *data)
2242 pci_free_irq_vectors(data);
2245 static void bcm4377_hci_free_dev(void *data)
2250 static void bcm4377_hci_unregister_dev(void *data)
2252 hci_unregister_dev(data);
2255 static int bcm4377_probe(struct pci_dev *pdev, const struct pci_device_id *id)
2257 struct bcm4377_data *bcm4377;
2258 struct hci_dev *hdev;
2261 ret = dma_set_mask_and_coherent(&pdev->dev, BCM4377_DMA_MASK);
2265 bcm4377 = devm_kzalloc(&pdev->dev, sizeof(*bcm4377), GFP_KERNEL);
2269 bcm4377->pdev = pdev;
2270 bcm4377->hw = &bcm4377_hw_variants[id->driver_data];
2271 init_completion(&bcm4377->event);
2273 ret = bcm4377_prepare_rings(bcm4377);
2277 ret = bcm4377_init_context(bcm4377);
2281 ret = bcm4377_probe_dmi(bcm4377);
2284 ret = bcm4377_probe_of(bcm4377);
2287 if (!bcm4377->board_type) {
2288 dev_err(&pdev->dev, "unable to determine board type\n");
2292 if (bcm4377->hw->disable_aspm)
2293 bcm4377_disable_aspm(bcm4377);
2295 ret = pci_reset_function_locked(pdev);
2299 "function level reset failed with %d; trying to continue anyway\n",
2303 * If this number is too low and we try to access any BAR too
2304 * early the device will crash. Experiments have shown that
2305 * approximately 50 msec is the minimum amount we have to wait.
2306 * Let's double that to be safe.
2310 ret = pcim_enable_device(pdev);
2313 pci_set_master(pdev);
2315 ret = bcm4377_init_cfg(bcm4377);
2319 bcm4377->bar0 = pcim_iomap(pdev, 0, 0);
2322 bcm4377->bar2 = pcim_iomap(pdev, 2, 0);
2326 ret = bcm4377_parse_otp(bcm4377);
2328 dev_err(&pdev->dev, "Reading OTP failed with %d\n", ret);
2333 * Legacy interrupts result in an IRQ storm because we don't know where
2334 * the interrupt mask and status registers for these chips are.
2335 * MSIs are acked automatically instead.
2337 ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI);
2340 ret = devm_add_action_or_reset(&pdev->dev, bcm4377_pci_free_irq_vectors,
2345 irq = pci_irq_vector(pdev, 0);
2349 ret = devm_request_irq(&pdev->dev, irq, bcm4377_irq, 0, "bcm4377",
2354 hdev = hci_alloc_dev();
2357 ret = devm_add_action_or_reset(&pdev->dev, bcm4377_hci_free_dev, hdev);
2361 bcm4377->hdev = hdev;
2363 hdev->bus = HCI_PCI;
2364 hdev->dev_type = HCI_PRIMARY;
2365 hdev->open = bcm4377_hci_open;
2366 hdev->close = bcm4377_hci_close;
2367 hdev->send = bcm4377_hci_send_frame;
2368 hdev->set_bdaddr = bcm4377_hci_set_bdaddr;
2369 hdev->setup = bcm4377_hci_setup;
2371 set_bit(HCI_QUIRK_USE_BDADDR_PROPERTY, &hdev->quirks);
2372 if (bcm4377->hw->broken_mws_transport_config)
2373 set_bit(HCI_QUIRK_BROKEN_MWS_TRANSPORT_CONFIG, &hdev->quirks);
2374 if (bcm4377->hw->broken_ext_scan)
2375 set_bit(HCI_QUIRK_BROKEN_EXT_SCAN, &hdev->quirks);
2376 if (bcm4377->hw->broken_le_coded)
2377 set_bit(HCI_QUIRK_BROKEN_LE_CODED, &hdev->quirks);
2379 pci_set_drvdata(pdev, bcm4377);
2380 hci_set_drvdata(hdev, bcm4377);
2381 SET_HCIDEV_DEV(hdev, &pdev->dev);
2383 ret = bcm4377_boot(bcm4377);
2387 ret = bcm4377_setup_rti(bcm4377);
2391 ret = hci_register_dev(hdev);
2394 return devm_add_action_or_reset(&pdev->dev, bcm4377_hci_unregister_dev,
2398 static int bcm4377_suspend(struct pci_dev *pdev, pm_message_t state)
2400 struct bcm4377_data *bcm4377 = pci_get_drvdata(pdev);
2403 ret = hci_suspend_dev(bcm4377->hdev);
2407 iowrite32(BCM4377_BAR0_SLEEP_CONTROL_QUIESCE,
2408 bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL);
2413 static int bcm4377_resume(struct pci_dev *pdev)
2415 struct bcm4377_data *bcm4377 = pci_get_drvdata(pdev);
2417 iowrite32(BCM4377_BAR0_SLEEP_CONTROL_UNQUIESCE,
2418 bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL);
2420 return hci_resume_dev(bcm4377->hdev);
2423 static const struct dmi_system_id bcm4377_dmi_board_table[] = {
2426 DMI_MATCH(DMI_BOARD_VENDOR, "Apple Inc."),
2427 DMI_MATCH(DMI_PRODUCT_NAME, "MacBookAir9,1"),
2429 .driver_data = "apple,formosa",
2433 DMI_MATCH(DMI_BOARD_VENDOR, "Apple Inc."),
2434 DMI_MATCH(DMI_PRODUCT_NAME, "MacBookPro15,4"),
2436 .driver_data = "apple,formosa",
2440 DMI_MATCH(DMI_BOARD_VENDOR, "Apple Inc."),
2441 DMI_MATCH(DMI_PRODUCT_NAME, "MacBookPro16,3"),
2443 .driver_data = "apple,formosa",
2448 static const struct bcm4377_hw bcm4377_hw_variants[] = {
2451 .otp_offset = 0x4120,
2452 .bar0_window1 = 0x1800b000,
2453 .bar0_window2 = 0x1810c000,
2454 .disable_aspm = true,
2455 .broken_ext_scan = true,
2456 .send_ptb = bcm4377_send_ptb,
2461 .otp_offset = 0x4120,
2462 .bar0_window1 = 0x18002000,
2463 .bar0_window2 = 0x1810a000,
2464 .bar0_core2_window2 = 0x18107000,
2465 .has_bar0_core2_window2 = true,
2466 .broken_mws_transport_config = true,
2467 .broken_le_coded = true,
2468 .send_calibration = bcm4378_send_calibration,
2469 .send_ptb = bcm4378_send_ptb,
2474 .otp_offset = 0x413c,
2475 .bar0_window1 = 0x18002000,
2476 .bar0_window2 = 0x18109000,
2477 .bar0_core2_window2 = 0x18106000,
2478 .has_bar0_core2_window2 = true,
2479 .clear_pciecfg_subsystem_ctrl_bit19 = true,
2480 .broken_mws_transport_config = true,
2481 .broken_le_coded = true,
2482 .send_calibration = bcm4387_send_calibration,
2483 .send_ptb = bcm4378_send_ptb,
2487 #define BCM4377_DEVID_ENTRY(id) \
2489 PCI_VENDOR_ID_BROADCOM, BCM##id##_DEVICE_ID, PCI_ANY_ID, \
2490 PCI_ANY_ID, PCI_CLASS_NETWORK_OTHER << 8, 0xffff00, \
2494 static const struct pci_device_id bcm4377_devid_table[] = {
2495 BCM4377_DEVID_ENTRY(4377),
2496 BCM4377_DEVID_ENTRY(4378),
2497 BCM4377_DEVID_ENTRY(4387),
2500 MODULE_DEVICE_TABLE(pci, bcm4377_devid_table);
2502 static struct pci_driver bcm4377_pci_driver = {
2503 .name = "hci_bcm4377",
2504 .id_table = bcm4377_devid_table,
2505 .probe = bcm4377_probe,
2506 .suspend = bcm4377_suspend,
2507 .resume = bcm4377_resume,
2509 module_pci_driver(bcm4377_pci_driver);
2511 MODULE_AUTHOR("Sven Peter <sven@svenpeter.dev>");
2512 MODULE_DESCRIPTION("Bluetooth support for Broadcom 4377/4378/4387 devices");
2513 MODULE_LICENSE("Dual MIT/GPL");
2514 MODULE_FIRMWARE("brcm/brcmbt4377*.bin");
2515 MODULE_FIRMWARE("brcm/brcmbt4377*.ptb");
2516 MODULE_FIRMWARE("brcm/brcmbt4378*.bin");
2517 MODULE_FIRMWARE("brcm/brcmbt4378*.ptb");
2518 MODULE_FIRMWARE("brcm/brcmbt4387*.bin");
2519 MODULE_FIRMWARE("brcm/brcmbt4387*.ptb");