1 // SPDX-License-Identifier: GPL-2.0+
3 * NVIDIA Tegra XUSB device mode controller
5 * Copyright (c) 2013-2019, NVIDIA CORPORATION. All rights reserved.
6 * Copyright (c) 2015, Google Inc.
10 #include <linux/completion.h>
11 #include <linux/delay.h>
12 #include <linux/dma-mapping.h>
13 #include <linux/dmapool.h>
14 #include <linux/interrupt.h>
15 #include <linux/iopoll.h>
16 #include <linux/kernel.h>
17 #include <linux/module.h>
19 #include <linux/of_device.h>
20 #include <linux/phy/phy.h>
21 #include <linux/phy/tegra/xusb.h>
22 #include <linux/pm_domain.h>
23 #include <linux/platform_device.h>
24 #include <linux/pm_runtime.h>
25 #include <linux/regulator/consumer.h>
26 #include <linux/reset.h>
27 #include <linux/usb/ch9.h>
28 #include <linux/usb/gadget.h>
29 #include <linux/usb/role.h>
30 #include <linux/workqueue.h>
32 /* XUSB_DEV registers */
34 #define SPARAM_ERSTMAX_MASK GENMASK(20, 16)
35 #define SPARAM_ERSTMAX(x) (((x) << 16) & SPARAM_ERSTMAX_MASK)
37 #define DB_TARGET_MASK GENMASK(15, 8)
38 #define DB_TARGET(x) (((x) << 8) & DB_TARGET_MASK)
39 #define DB_STREAMID_MASK GENMASK(31, 16)
40 #define DB_STREAMID(x) (((x) << 16) & DB_STREAMID_MASK)
42 #define ERSTSZ_ERSTXSZ_SHIFT(x) ((x) * 16)
43 #define ERSTSZ_ERSTXSZ_MASK GENMASK(15, 0)
44 #define ERSTXBALO(x) (0x010 + 8 * (x))
45 #define ERSTXBAHI(x) (0x014 + 8 * (x))
47 #define ERDPLO_EHB BIT(3)
50 #define EREPLO_ECS BIT(0)
51 #define EREPLO_SEGI BIT(1)
54 #define CTRL_RUN BIT(0)
55 #define CTRL_LSE BIT(1)
56 #define CTRL_IE BIT(4)
57 #define CTRL_SMI_EVT BIT(5)
58 #define CTRL_SMI_DSE BIT(6)
59 #define CTRL_EWE BIT(7)
60 #define CTRL_DEVADDR_MASK GENMASK(30, 24)
61 #define CTRL_DEVADDR(x) (((x) << 24) & CTRL_DEVADDR_MASK)
62 #define CTRL_ENABLE BIT(31)
67 #define RT_IMOD_IMODI_MASK GENMASK(15, 0)
68 #define RT_IMOD_IMODI(x) ((x) & RT_IMOD_IMODI_MASK)
69 #define RT_IMOD_IMODC_MASK GENMASK(31, 16)
70 #define RT_IMOD_IMODC(x) (((x) << 16) & RT_IMOD_IMODC_MASK)
72 #define PORTSC_CCS BIT(0)
73 #define PORTSC_PED BIT(1)
74 #define PORTSC_PR BIT(4)
75 #define PORTSC_PLS_SHIFT 5
76 #define PORTSC_PLS_MASK GENMASK(8, 5)
77 #define PORTSC_PLS_U0 0x0
78 #define PORTSC_PLS_U2 0x2
79 #define PORTSC_PLS_U3 0x3
80 #define PORTSC_PLS_DISABLED 0x4
81 #define PORTSC_PLS_RXDETECT 0x5
82 #define PORTSC_PLS_INACTIVE 0x6
83 #define PORTSC_PLS_RESUME 0xf
84 #define PORTSC_PLS(x) (((x) << PORTSC_PLS_SHIFT) & PORTSC_PLS_MASK)
85 #define PORTSC_PS_SHIFT 10
86 #define PORTSC_PS_MASK GENMASK(13, 10)
87 #define PORTSC_PS_UNDEFINED 0x0
88 #define PORTSC_PS_FS 0x1
89 #define PORTSC_PS_LS 0x2
90 #define PORTSC_PS_HS 0x3
91 #define PORTSC_PS_SS 0x4
92 #define PORTSC_LWS BIT(16)
93 #define PORTSC_CSC BIT(17)
94 #define PORTSC_WRC BIT(19)
95 #define PORTSC_PRC BIT(21)
96 #define PORTSC_PLC BIT(22)
97 #define PORTSC_CEC BIT(23)
98 #define PORTSC_WPR BIT(30)
99 #define PORTSC_CHANGE_MASK (PORTSC_CSC | PORTSC_WRC | PORTSC_PRC | \
100 PORTSC_PLC | PORTSC_CEC)
103 #define MFINDEX 0x048
104 #define MFINDEX_FRAME_SHIFT 3
105 #define MFINDEX_FRAME_MASK GENMASK(13, 3)
107 #define PORTPM_L1S_MASK GENMASK(1, 0)
108 #define PORTPM_L1S_DROP 0x0
109 #define PORTPM_L1S_ACCEPT 0x1
110 #define PORTPM_L1S_NYET 0x2
111 #define PORTPM_L1S_STALL 0x3
112 #define PORTPM_L1S(x) ((x) & PORTPM_L1S_MASK)
113 #define PORTPM_RWE BIT(3)
114 #define PORTPM_U2TIMEOUT_MASK GENMASK(15, 8)
115 #define PORTPM_U1TIMEOUT_MASK GENMASK(23, 16)
116 #define PORTPM_FLA BIT(24)
117 #define PORTPM_VBA BIT(25)
118 #define PORTPM_WOC BIT(26)
119 #define PORTPM_WOD BIT(27)
120 #define PORTPM_U1E BIT(28)
121 #define PORTPM_U2E BIT(29)
122 #define PORTPM_FRWE BIT(30)
123 #define PORTPM_PNG_CYA BIT(31)
124 #define EP_HALT 0x050
125 #define EP_PAUSE 0x054
126 #define EP_RELOAD 0x058
127 #define EP_STCHG 0x05c
128 #define DEVNOTIF_LO 0x064
129 #define DEVNOTIF_LO_TRIG BIT(0)
130 #define DEVNOTIF_LO_TYPE_MASK GENMASK(7, 4)
131 #define DEVNOTIF_LO_TYPE(x) (((x) << 4) & DEVNOTIF_LO_TYPE_MASK)
132 #define DEVNOTIF_LO_TYPE_FUNCTION_WAKE 0x1
133 #define DEVNOTIF_HI 0x068
134 #define PORTHALT 0x06c
135 #define PORTHALT_HALT_LTSSM BIT(0)
136 #define PORTHALT_HALT_REJECT BIT(1)
137 #define PORTHALT_STCHG_REQ BIT(20)
138 #define PORTHALT_STCHG_INTR_EN BIT(24)
139 #define PORT_TM 0x070
140 #define EP_THREAD_ACTIVE 0x074
141 #define EP_STOPPED 0x078
142 #define HSFSPI_COUNT0 0x100
143 #define HSFSPI_COUNT13 0x134
144 #define HSFSPI_COUNT13_U2_RESUME_K_DURATION_MASK GENMASK(29, 0)
145 #define HSFSPI_COUNT13_U2_RESUME_K_DURATION(x) ((x) & \
146 HSFSPI_COUNT13_U2_RESUME_K_DURATION_MASK)
148 #define SSPX_CORE_CNT0 0x610
149 #define SSPX_CORE_CNT0_PING_TBURST_MASK GENMASK(7, 0)
150 #define SSPX_CORE_CNT0_PING_TBURST(x) ((x) & SSPX_CORE_CNT0_PING_TBURST_MASK)
151 #define SSPX_CORE_CNT30 0x688
152 #define SSPX_CORE_CNT30_LMPITP_TIMER_MASK GENMASK(19, 0)
153 #define SSPX_CORE_CNT30_LMPITP_TIMER(x) ((x) & \
154 SSPX_CORE_CNT30_LMPITP_TIMER_MASK)
155 #define SSPX_CORE_CNT32 0x690
156 #define SSPX_CORE_CNT32_POLL_TBURST_MAX_MASK GENMASK(7, 0)
157 #define SSPX_CORE_CNT32_POLL_TBURST_MAX(x) ((x) & \
158 SSPX_CORE_CNT32_POLL_TBURST_MAX_MASK)
159 #define SSPX_CORE_PADCTL4 0x750
160 #define SSPX_CORE_PADCTL4_RXDAT_VLD_TIMEOUT_U3_MASK GENMASK(19, 0)
161 #define SSPX_CORE_PADCTL4_RXDAT_VLD_TIMEOUT_U3(x) ((x) & \
162 SSPX_CORE_PADCTL4_RXDAT_VLD_TIMEOUT_U3_MASK)
163 #define BLCG_DFPCI BIT(0)
164 #define BLCG_UFPCI BIT(1)
165 #define BLCG_FE BIT(2)
166 #define BLCG_COREPLL_PWRDN BIT(8)
167 #define BLCG_IOPLL_0_PWRDN BIT(9)
168 #define BLCG_IOPLL_1_PWRDN BIT(10)
169 #define BLCG_IOPLL_2_PWRDN BIT(11)
170 #define BLCG_ALL 0x1ff
171 #define CFG_DEV_SSPI_XFER 0x858
172 #define CFG_DEV_SSPI_XFER_ACKTIMEOUT_MASK GENMASK(31, 0)
173 #define CFG_DEV_SSPI_XFER_ACKTIMEOUT(x) ((x) & \
174 CFG_DEV_SSPI_XFER_ACKTIMEOUT_MASK)
175 #define CFG_DEV_FE 0x85c
176 #define CFG_DEV_FE_PORTREGSEL_MASK GENMASK(1, 0)
177 #define CFG_DEV_FE_PORTREGSEL_SS_PI 1
178 #define CFG_DEV_FE_PORTREGSEL_HSFS_PI 2
179 #define CFG_DEV_FE_PORTREGSEL(x) ((x) & CFG_DEV_FE_PORTREGSEL_MASK)
180 #define CFG_DEV_FE_INFINITE_SS_RETRY BIT(29)
183 #define XUSB_DEV_CFG_1 0x004
184 #define XUSB_DEV_CFG_1_IO_SPACE_EN BIT(0)
185 #define XUSB_DEV_CFG_1_MEMORY_SPACE_EN BIT(1)
186 #define XUSB_DEV_CFG_1_BUS_MASTER_EN BIT(2)
187 #define XUSB_DEV_CFG_4 0x010
188 #define XUSB_DEV_CFG_4_BASE_ADDR_MASK GENMASK(31, 15)
189 #define XUSB_DEV_CFG_5 0x014
192 #define XUSB_DEV_CONFIGURATION_0 0x180
193 #define XUSB_DEV_CONFIGURATION_0_EN_FPCI BIT(0)
194 #define XUSB_DEV_INTR_MASK_0 0x188
195 #define XUSB_DEV_INTR_MASK_0_IP_INT_MASK BIT(16)
197 struct tegra_xudc_ep_context {
206 #define EP_STATE_DISABLED 0
207 #define EP_STATE_RUNNING 1
208 #define EP_STATE_HALTED 2
209 #define EP_STATE_STOPPED 3
210 #define EP_STATE_ERROR 4
212 #define EP_TYPE_INVALID 0
213 #define EP_TYPE_ISOCH_OUT 1
214 #define EP_TYPE_BULK_OUT 2
215 #define EP_TYPE_INTERRUPT_OUT 3
216 #define EP_TYPE_CONTROL 4
217 #define EP_TYPE_ISCOH_IN 5
218 #define EP_TYPE_BULK_IN 6
219 #define EP_TYPE_INTERRUPT_IN 7
221 #define BUILD_EP_CONTEXT_RW(name, member, shift, mask) \
222 static inline u32 ep_ctx_read_##name(struct tegra_xudc_ep_context *ctx) \
224 return (le32_to_cpu(ctx->member) >> (shift)) & (mask); \
227 ep_ctx_write_##name(struct tegra_xudc_ep_context *ctx, u32 val) \
231 tmp = le32_to_cpu(ctx->member) & ~((mask) << (shift)); \
232 tmp |= (val & (mask)) << (shift); \
233 ctx->member = cpu_to_le32(tmp); \
236 BUILD_EP_CONTEXT_RW(state, info0, 0, 0x7)
237 BUILD_EP_CONTEXT_RW(mult, info0, 8, 0x3)
238 BUILD_EP_CONTEXT_RW(max_pstreams, info0, 10, 0x1f)
239 BUILD_EP_CONTEXT_RW(lsa, info0, 15, 0x1)
240 BUILD_EP_CONTEXT_RW(interval, info0, 16, 0xff)
241 BUILD_EP_CONTEXT_RW(cerr, info1, 1, 0x3)
242 BUILD_EP_CONTEXT_RW(type, info1, 3, 0x7)
243 BUILD_EP_CONTEXT_RW(hid, info1, 7, 0x1)
244 BUILD_EP_CONTEXT_RW(max_burst_size, info1, 8, 0xff)
245 BUILD_EP_CONTEXT_RW(max_packet_size, info1, 16, 0xffff)
246 BUILD_EP_CONTEXT_RW(dcs, deq_lo, 0, 0x1)
247 BUILD_EP_CONTEXT_RW(deq_lo, deq_lo, 4, 0xfffffff)
248 BUILD_EP_CONTEXT_RW(deq_hi, deq_hi, 0, 0xffffffff)
249 BUILD_EP_CONTEXT_RW(avg_trb_len, tx_info, 0, 0xffff)
250 BUILD_EP_CONTEXT_RW(max_esit_payload, tx_info, 16, 0xffff)
251 BUILD_EP_CONTEXT_RW(edtla, rsvd[0], 0, 0xffffff)
252 BUILD_EP_CONTEXT_RW(seq_num, rsvd[0], 24, 0xff)
253 BUILD_EP_CONTEXT_RW(partial_td, rsvd[0], 25, 0x1)
254 BUILD_EP_CONTEXT_RW(cerrcnt, rsvd[1], 18, 0x3)
255 BUILD_EP_CONTEXT_RW(data_offset, rsvd[2], 0, 0x1ffff)
256 BUILD_EP_CONTEXT_RW(numtrbs, rsvd[2], 22, 0x1f)
257 BUILD_EP_CONTEXT_RW(devaddr, rsvd[6], 0, 0x7f)
259 static inline u64 ep_ctx_read_deq_ptr(struct tegra_xudc_ep_context *ctx)
261 return ((u64)ep_ctx_read_deq_hi(ctx) << 32) |
262 (ep_ctx_read_deq_lo(ctx) << 4);
266 ep_ctx_write_deq_ptr(struct tegra_xudc_ep_context *ctx, u64 addr)
268 ep_ctx_write_deq_lo(ctx, lower_32_bits(addr) >> 4);
269 ep_ctx_write_deq_hi(ctx, upper_32_bits(addr));
272 struct tegra_xudc_trb {
279 #define TRB_TYPE_RSVD 0
280 #define TRB_TYPE_NORMAL 1
281 #define TRB_TYPE_SETUP_STAGE 2
282 #define TRB_TYPE_DATA_STAGE 3
283 #define TRB_TYPE_STATUS_STAGE 4
284 #define TRB_TYPE_ISOCH 5
285 #define TRB_TYPE_LINK 6
286 #define TRB_TYPE_TRANSFER_EVENT 32
287 #define TRB_TYPE_PORT_STATUS_CHANGE_EVENT 34
288 #define TRB_TYPE_STREAM 48
289 #define TRB_TYPE_SETUP_PACKET_EVENT 63
291 #define TRB_CMPL_CODE_INVALID 0
292 #define TRB_CMPL_CODE_SUCCESS 1
293 #define TRB_CMPL_CODE_DATA_BUFFER_ERR 2
294 #define TRB_CMPL_CODE_BABBLE_DETECTED_ERR 3
295 #define TRB_CMPL_CODE_USB_TRANS_ERR 4
296 #define TRB_CMPL_CODE_TRB_ERR 5
297 #define TRB_CMPL_CODE_STALL 6
298 #define TRB_CMPL_CODE_INVALID_STREAM_TYPE_ERR 10
299 #define TRB_CMPL_CODE_SHORT_PACKET 13
300 #define TRB_CMPL_CODE_RING_UNDERRUN 14
301 #define TRB_CMPL_CODE_RING_OVERRUN 15
302 #define TRB_CMPL_CODE_EVENT_RING_FULL_ERR 21
303 #define TRB_CMPL_CODE_STOPPED 26
304 #define TRB_CMPL_CODE_ISOCH_BUFFER_OVERRUN 31
305 #define TRB_CMPL_CODE_STREAM_NUMP_ERROR 219
306 #define TRB_CMPL_CODE_PRIME_PIPE_RECEIVED 220
307 #define TRB_CMPL_CODE_HOST_REJECTED 221
308 #define TRB_CMPL_CODE_CTRL_DIR_ERR 222
309 #define TRB_CMPL_CODE_CTRL_SEQNUM_ERR 223
311 #define BUILD_TRB_RW(name, member, shift, mask) \
312 static inline u32 trb_read_##name(struct tegra_xudc_trb *trb) \
314 return (le32_to_cpu(trb->member) >> (shift)) & (mask); \
317 trb_write_##name(struct tegra_xudc_trb *trb, u32 val) \
321 tmp = le32_to_cpu(trb->member) & ~((mask) << (shift)); \
322 tmp |= (val & (mask)) << (shift); \
323 trb->member = cpu_to_le32(tmp); \
326 BUILD_TRB_RW(data_lo, data_lo, 0, 0xffffffff)
327 BUILD_TRB_RW(data_hi, data_hi, 0, 0xffffffff)
328 BUILD_TRB_RW(seq_num, status, 0, 0xffff)
329 BUILD_TRB_RW(transfer_len, status, 0, 0xffffff)
330 BUILD_TRB_RW(td_size, status, 17, 0x1f)
331 BUILD_TRB_RW(cmpl_code, status, 24, 0xff)
332 BUILD_TRB_RW(cycle, control, 0, 0x1)
333 BUILD_TRB_RW(toggle_cycle, control, 1, 0x1)
334 BUILD_TRB_RW(isp, control, 2, 0x1)
335 BUILD_TRB_RW(chain, control, 4, 0x1)
336 BUILD_TRB_RW(ioc, control, 5, 0x1)
337 BUILD_TRB_RW(type, control, 10, 0x3f)
338 BUILD_TRB_RW(stream_id, control, 16, 0xffff)
339 BUILD_TRB_RW(endpoint_id, control, 16, 0x1f)
340 BUILD_TRB_RW(tlbpc, control, 16, 0xf)
341 BUILD_TRB_RW(data_stage_dir, control, 16, 0x1)
342 BUILD_TRB_RW(frame_id, control, 20, 0x7ff)
343 BUILD_TRB_RW(sia, control, 31, 0x1)
345 static inline u64 trb_read_data_ptr(struct tegra_xudc_trb *trb)
347 return ((u64)trb_read_data_hi(trb) << 32) |
348 trb_read_data_lo(trb);
351 static inline void trb_write_data_ptr(struct tegra_xudc_trb *trb, u64 addr)
353 trb_write_data_lo(trb, lower_32_bits(addr));
354 trb_write_data_hi(trb, upper_32_bits(addr));
357 struct tegra_xudc_request {
358 struct usb_request usb_req;
361 unsigned int trbs_queued;
362 unsigned int trbs_needed;
365 struct tegra_xudc_trb *first_trb;
366 struct tegra_xudc_trb *last_trb;
368 struct list_head list;
371 struct tegra_xudc_ep {
372 struct tegra_xudc *xudc;
373 struct usb_ep usb_ep;
377 struct tegra_xudc_ep_context *context;
379 #define XUDC_TRANSFER_RING_SIZE 64
380 struct tegra_xudc_trb *transfer_ring;
381 dma_addr_t transfer_ring_phys;
383 unsigned int enq_ptr;
384 unsigned int deq_ptr;
387 bool stream_rejected;
389 struct list_head queue;
390 const struct usb_endpoint_descriptor *desc;
391 const struct usb_ss_ep_comp_descriptor *comp_desc;
394 struct tegra_xudc_sel_timing {
401 enum tegra_xudc_setup_state {
409 struct tegra_xudc_setup_packet {
410 struct usb_ctrlrequest ctrl_req;
411 unsigned int seq_num;
414 struct tegra_xudc_save_regs {
421 const struct tegra_xudc_soc *soc;
422 struct tegra_xusb_padctl *padctl;
426 struct usb_gadget gadget;
427 struct usb_gadget_driver *driver;
429 #define XUDC_NR_EVENT_RINGS 2
430 #define XUDC_EVENT_RING_SIZE 4096
431 struct tegra_xudc_trb *event_ring[XUDC_NR_EVENT_RINGS];
432 dma_addr_t event_ring_phys[XUDC_NR_EVENT_RINGS];
433 unsigned int event_ring_index;
434 unsigned int event_ring_deq_ptr;
437 #define XUDC_NR_EPS 32
438 struct tegra_xudc_ep ep[XUDC_NR_EPS];
439 struct tegra_xudc_ep_context *ep_context;
440 dma_addr_t ep_context_phys;
442 struct device *genpd_dev_device;
443 struct device *genpd_dev_ss;
444 struct device_link *genpd_dl_device;
445 struct device_link *genpd_dl_ss;
447 struct dma_pool *transfer_ring_pool;
449 bool queued_setup_packet;
450 struct tegra_xudc_setup_packet setup_packet;
451 enum tegra_xudc_setup_state setup_state;
456 struct tegra_xudc_sel_timing sel_timing;
457 u8 test_mode_pattern;
459 struct tegra_xudc_request *ep0_req;
463 unsigned int nr_enabled_eps;
464 unsigned int nr_isoch_eps;
466 unsigned int device_state;
467 unsigned int resume_state;
472 resource_size_t phys_base;
476 struct regulator_bulk_data *supplies;
478 struct clk_bulk_data *clks;
480 enum usb_role device_mode;
481 struct usb_role_switch *usb_role_sw;
482 struct work_struct usb_role_sw_work;
484 struct phy *usb3_phy;
485 struct phy *utmi_phy;
487 struct tegra_xudc_save_regs saved_regs;
491 struct completion disconnect_complete;
495 #define TOGGLE_VBUS_WAIT_MS 100
496 struct delayed_work plc_reset_work;
499 struct delayed_work port_reset_war_work;
500 bool wait_for_sec_prc;
503 #define XUDC_TRB_MAX_BUFFER_SIZE 65536
504 #define XUDC_MAX_ISOCH_EPS 4
505 #define XUDC_INTERRUPT_MODERATION_US 0
507 static struct usb_endpoint_descriptor tegra_xudc_ep0_desc = {
508 .bLength = USB_DT_ENDPOINT_SIZE,
509 .bDescriptorType = USB_DT_ENDPOINT,
510 .bEndpointAddress = 0,
511 .bmAttributes = USB_ENDPOINT_XFER_CONTROL,
512 .wMaxPacketSize = cpu_to_le16(64),
515 struct tegra_xudc_soc {
516 const char * const *supply_names;
517 unsigned int num_supplies;
518 const char * const *clock_names;
519 unsigned int num_clks;
523 bool invalid_seq_num;
525 bool port_reset_quirk;
529 static inline u32 fpci_readl(struct tegra_xudc *xudc, unsigned int offset)
531 return readl(xudc->fpci + offset);
534 static inline void fpci_writel(struct tegra_xudc *xudc, u32 val,
537 writel(val, xudc->fpci + offset);
540 static inline u32 ipfs_readl(struct tegra_xudc *xudc, unsigned int offset)
542 return readl(xudc->ipfs + offset);
545 static inline void ipfs_writel(struct tegra_xudc *xudc, u32 val,
548 writel(val, xudc->ipfs + offset);
551 static inline u32 xudc_readl(struct tegra_xudc *xudc, unsigned int offset)
553 return readl(xudc->base + offset);
556 static inline void xudc_writel(struct tegra_xudc *xudc, u32 val,
559 writel(val, xudc->base + offset);
562 static inline int xudc_readl_poll(struct tegra_xudc *xudc,
563 unsigned int offset, u32 mask, u32 val)
567 return readl_poll_timeout_atomic(xudc->base + offset, regval,
568 (regval & mask) == val, 1, 100);
571 static inline struct tegra_xudc *to_xudc(struct usb_gadget *gadget)
573 return container_of(gadget, struct tegra_xudc, gadget);
576 static inline struct tegra_xudc_ep *to_xudc_ep(struct usb_ep *ep)
578 return container_of(ep, struct tegra_xudc_ep, usb_ep);
581 static inline struct tegra_xudc_request *to_xudc_req(struct usb_request *req)
583 return container_of(req, struct tegra_xudc_request, usb_req);
586 static inline void dump_trb(struct tegra_xudc *xudc, const char *type,
587 struct tegra_xudc_trb *trb)
590 "%s: %p, lo = %#x, hi = %#x, status = %#x, control = %#x\n",
591 type, trb, trb->data_lo, trb->data_hi, trb->status,
595 static void tegra_xudc_device_mode_on(struct tegra_xudc *xudc)
599 pm_runtime_get_sync(xudc->dev);
601 err = phy_power_on(xudc->utmi_phy);
603 dev_err(xudc->dev, "utmi power on failed %d\n", err);
605 err = phy_power_on(xudc->usb3_phy);
607 dev_err(xudc->dev, "usb3 phy power on failed %d\n", err);
609 dev_dbg(xudc->dev, "device mode on\n");
611 tegra_xusb_padctl_set_vbus_override(xudc->padctl, true);
613 xudc->device_mode = USB_ROLE_DEVICE;
616 static void tegra_xudc_device_mode_off(struct tegra_xudc *xudc)
618 bool connected = false;
622 dev_dbg(xudc->dev, "device mode off\n");
624 connected = !!(xudc_readl(xudc, PORTSC) & PORTSC_CCS);
626 reinit_completion(&xudc->disconnect_complete);
628 tegra_xusb_padctl_set_vbus_override(xudc->padctl, false);
630 pls = (xudc_readl(xudc, PORTSC) & PORTSC_PLS_MASK) >>
633 /* Direct link to U0 if disconnected in RESUME or U2. */
634 if (xudc->soc->pls_quirk && xudc->gadget.speed == USB_SPEED_SUPER &&
635 (pls == PORTSC_PLS_RESUME || pls == PORTSC_PLS_U2)) {
636 val = xudc_readl(xudc, PORTPM);
638 xudc_writel(xudc, val, PORTPM);
640 val = xudc_readl(xudc, PORTSC);
641 val &= ~(PORTSC_CHANGE_MASK | PORTSC_PLS_MASK);
642 val |= PORTSC_LWS | PORTSC_PLS(PORTSC_PLS_U0);
643 xudc_writel(xudc, val, PORTSC);
646 xudc->device_mode = USB_ROLE_NONE;
648 /* Wait for disconnect event. */
650 wait_for_completion(&xudc->disconnect_complete);
652 /* Make sure interrupt handler has completed before powergating. */
653 synchronize_irq(xudc->irq);
655 err = phy_power_off(xudc->utmi_phy);
657 dev_err(xudc->dev, "utmi_phy power off failed %d\n", err);
659 err = phy_power_off(xudc->usb3_phy);
661 dev_err(xudc->dev, "usb3_phy power off failed %d\n", err);
663 pm_runtime_put(xudc->dev);
666 static void tegra_xudc_usb_role_sw_work(struct work_struct *work)
668 struct tegra_xudc *xudc = container_of(work, struct tegra_xudc,
671 if (!xudc->usb_role_sw ||
672 usb_role_switch_get_role(xudc->usb_role_sw) == USB_ROLE_DEVICE)
673 tegra_xudc_device_mode_on(xudc);
675 tegra_xudc_device_mode_off(xudc);
679 static int tegra_xudc_usb_role_sw_set(struct device *dev, enum usb_role role)
681 struct tegra_xudc *xudc = dev_get_drvdata(dev);
684 dev_dbg(dev, "%s role is %d\n", __func__, role);
686 spin_lock_irqsave(&xudc->lock, flags);
688 if (!xudc->suspended)
689 schedule_work(&xudc->usb_role_sw_work);
691 spin_unlock_irqrestore(&xudc->lock, flags);
696 static void tegra_xudc_plc_reset_work(struct work_struct *work)
698 struct delayed_work *dwork = to_delayed_work(work);
699 struct tegra_xudc *xudc = container_of(dwork, struct tegra_xudc,
703 spin_lock_irqsave(&xudc->lock, flags);
705 if (xudc->wait_csc) {
706 u32 pls = (xudc_readl(xudc, PORTSC) & PORTSC_PLS_MASK) >>
709 if (pls == PORTSC_PLS_INACTIVE) {
710 dev_info(xudc->dev, "PLS = Inactive. Toggle VBUS\n");
711 tegra_xusb_padctl_set_vbus_override(xudc->padctl,
713 tegra_xusb_padctl_set_vbus_override(xudc->padctl, true);
714 xudc->wait_csc = false;
718 spin_unlock_irqrestore(&xudc->lock, flags);
721 static void tegra_xudc_port_reset_war_work(struct work_struct *work)
723 struct delayed_work *dwork = to_delayed_work(work);
724 struct tegra_xudc *xudc =
725 container_of(dwork, struct tegra_xudc, port_reset_war_work);
730 spin_lock_irqsave(&xudc->lock, flags);
732 if ((xudc->device_mode == USB_ROLE_DEVICE)
733 && xudc->wait_for_sec_prc) {
734 pls = (xudc_readl(xudc, PORTSC) & PORTSC_PLS_MASK) >>
736 dev_dbg(xudc->dev, "pls = %x\n", pls);
738 if (pls == PORTSC_PLS_DISABLED) {
739 dev_dbg(xudc->dev, "toggle vbus\n");
740 /* PRC doesn't complete in 100ms, toggle the vbus */
741 ret = tegra_phy_xusb_utmi_port_reset(xudc->utmi_phy);
743 xudc->wait_for_sec_prc = 0;
747 spin_unlock_irqrestore(&xudc->lock, flags);
750 static dma_addr_t trb_virt_to_phys(struct tegra_xudc_ep *ep,
751 struct tegra_xudc_trb *trb)
755 index = trb - ep->transfer_ring;
757 if (WARN_ON(index >= XUDC_TRANSFER_RING_SIZE))
760 return (ep->transfer_ring_phys + index * sizeof(*trb));
763 static struct tegra_xudc_trb *trb_phys_to_virt(struct tegra_xudc_ep *ep,
766 struct tegra_xudc_trb *trb;
769 index = (addr - ep->transfer_ring_phys) / sizeof(*trb);
771 if (WARN_ON(index >= XUDC_TRANSFER_RING_SIZE))
774 trb = &ep->transfer_ring[index];
779 static void ep_reload(struct tegra_xudc *xudc, unsigned int ep)
781 xudc_writel(xudc, BIT(ep), EP_RELOAD);
782 xudc_readl_poll(xudc, EP_RELOAD, BIT(ep), 0);
785 static void ep_pause(struct tegra_xudc *xudc, unsigned int ep)
789 val = xudc_readl(xudc, EP_PAUSE);
794 xudc_writel(xudc, val, EP_PAUSE);
796 xudc_readl_poll(xudc, EP_STCHG, BIT(ep), BIT(ep));
798 xudc_writel(xudc, BIT(ep), EP_STCHG);
801 static void ep_unpause(struct tegra_xudc *xudc, unsigned int ep)
805 val = xudc_readl(xudc, EP_PAUSE);
806 if (!(val & BIT(ep)))
810 xudc_writel(xudc, val, EP_PAUSE);
812 xudc_readl_poll(xudc, EP_STCHG, BIT(ep), BIT(ep));
814 xudc_writel(xudc, BIT(ep), EP_STCHG);
817 static void ep_unpause_all(struct tegra_xudc *xudc)
821 val = xudc_readl(xudc, EP_PAUSE);
823 xudc_writel(xudc, 0, EP_PAUSE);
825 xudc_readl_poll(xudc, EP_STCHG, val, val);
827 xudc_writel(xudc, val, EP_STCHG);
830 static void ep_halt(struct tegra_xudc *xudc, unsigned int ep)
834 val = xudc_readl(xudc, EP_HALT);
838 xudc_writel(xudc, val, EP_HALT);
840 xudc_readl_poll(xudc, EP_STCHG, BIT(ep), BIT(ep));
842 xudc_writel(xudc, BIT(ep), EP_STCHG);
845 static void ep_unhalt(struct tegra_xudc *xudc, unsigned int ep)
849 val = xudc_readl(xudc, EP_HALT);
850 if (!(val & BIT(ep)))
853 xudc_writel(xudc, val, EP_HALT);
855 xudc_readl_poll(xudc, EP_STCHG, BIT(ep), BIT(ep));
857 xudc_writel(xudc, BIT(ep), EP_STCHG);
860 static void ep_unhalt_all(struct tegra_xudc *xudc)
864 val = xudc_readl(xudc, EP_HALT);
867 xudc_writel(xudc, 0, EP_HALT);
869 xudc_readl_poll(xudc, EP_STCHG, val, val);
871 xudc_writel(xudc, val, EP_STCHG);
874 static void ep_wait_for_stopped(struct tegra_xudc *xudc, unsigned int ep)
876 xudc_readl_poll(xudc, EP_STOPPED, BIT(ep), BIT(ep));
877 xudc_writel(xudc, BIT(ep), EP_STOPPED);
880 static void ep_wait_for_inactive(struct tegra_xudc *xudc, unsigned int ep)
882 xudc_readl_poll(xudc, EP_THREAD_ACTIVE, BIT(ep), 0);
885 static void tegra_xudc_req_done(struct tegra_xudc_ep *ep,
886 struct tegra_xudc_request *req, int status)
888 struct tegra_xudc *xudc = ep->xudc;
890 dev_dbg(xudc->dev, "completing request %p on EP %u with status %d\n",
891 req, ep->index, status);
893 if (likely(req->usb_req.status == -EINPROGRESS))
894 req->usb_req.status = status;
896 list_del_init(&req->list);
898 if (usb_endpoint_xfer_control(ep->desc)) {
899 usb_gadget_unmap_request(&xudc->gadget, &req->usb_req,
900 (xudc->setup_state ==
903 usb_gadget_unmap_request(&xudc->gadget, &req->usb_req,
904 usb_endpoint_dir_in(ep->desc));
907 spin_unlock(&xudc->lock);
908 usb_gadget_giveback_request(&ep->usb_ep, &req->usb_req);
909 spin_lock(&xudc->lock);
912 static void tegra_xudc_ep_nuke(struct tegra_xudc_ep *ep, int status)
914 struct tegra_xudc_request *req;
916 while (!list_empty(&ep->queue)) {
917 req = list_first_entry(&ep->queue, struct tegra_xudc_request,
919 tegra_xudc_req_done(ep, req, status);
923 static unsigned int ep_available_trbs(struct tegra_xudc_ep *ep)
928 if (ep->deq_ptr > ep->enq_ptr)
929 return ep->deq_ptr - ep->enq_ptr - 1;
931 return XUDC_TRANSFER_RING_SIZE - (ep->enq_ptr - ep->deq_ptr) - 2;
934 static void tegra_xudc_queue_one_trb(struct tegra_xudc_ep *ep,
935 struct tegra_xudc_request *req,
936 struct tegra_xudc_trb *trb,
939 struct tegra_xudc *xudc = ep->xudc;
943 len = min_t(size_t, XUDC_TRB_MAX_BUFFER_SIZE, req->usb_req.length -
946 buf_addr = req->usb_req.dma + req->buf_queued;
950 trb_write_data_ptr(trb, buf_addr);
952 trb_write_transfer_len(trb, len);
953 trb_write_td_size(trb, req->trbs_needed - req->trbs_queued - 1);
955 if (req->trbs_queued == req->trbs_needed - 1 ||
956 (req->need_zlp && req->trbs_queued == req->trbs_needed - 2))
957 trb_write_chain(trb, 0);
959 trb_write_chain(trb, 1);
961 trb_write_ioc(trb, ioc);
963 if (usb_endpoint_dir_out(ep->desc) ||
964 (usb_endpoint_xfer_control(ep->desc) &&
965 (xudc->setup_state == DATA_STAGE_RECV)))
966 trb_write_isp(trb, 1);
968 trb_write_isp(trb, 0);
970 if (usb_endpoint_xfer_control(ep->desc)) {
971 if (xudc->setup_state == DATA_STAGE_XFER ||
972 xudc->setup_state == DATA_STAGE_RECV)
973 trb_write_type(trb, TRB_TYPE_DATA_STAGE);
975 trb_write_type(trb, TRB_TYPE_STATUS_STAGE);
977 if (xudc->setup_state == DATA_STAGE_XFER ||
978 xudc->setup_state == STATUS_STAGE_XFER)
979 trb_write_data_stage_dir(trb, 1);
981 trb_write_data_stage_dir(trb, 0);
982 } else if (usb_endpoint_xfer_isoc(ep->desc)) {
983 trb_write_type(trb, TRB_TYPE_ISOCH);
984 trb_write_sia(trb, 1);
985 trb_write_frame_id(trb, 0);
986 trb_write_tlbpc(trb, 0);
987 } else if (usb_ss_max_streams(ep->comp_desc)) {
988 trb_write_type(trb, TRB_TYPE_STREAM);
989 trb_write_stream_id(trb, req->usb_req.stream_id);
991 trb_write_type(trb, TRB_TYPE_NORMAL);
992 trb_write_stream_id(trb, 0);
995 trb_write_cycle(trb, ep->pcs);
998 req->buf_queued += len;
1000 dump_trb(xudc, "TRANSFER", trb);
1003 static unsigned int tegra_xudc_queue_trbs(struct tegra_xudc_ep *ep,
1004 struct tegra_xudc_request *req)
1006 unsigned int i, count, available;
1007 bool wait_td = false;
1009 available = ep_available_trbs(ep);
1010 count = req->trbs_needed - req->trbs_queued;
1011 if (available < count) {
1013 ep->ring_full = true;
1017 * To generate zero-length packet on USB bus, SW needs schedule a
1018 * standalone zero-length TD. According to HW's behavior, SW needs
1019 * to schedule TDs in different ways for different endpoint types.
1021 * For control endpoint:
1022 * - Data stage TD (IOC = 1, CH = 0)
1023 * - Ring doorbell and wait transfer event
1024 * - Data stage TD for ZLP (IOC = 1, CH = 0)
1027 * For bulk and interrupt endpoints:
1028 * - Normal transfer TD (IOC = 0, CH = 0)
1029 * - Normal transfer TD for ZLP (IOC = 1, CH = 0)
1033 if (req->need_zlp && usb_endpoint_xfer_control(ep->desc) && count > 1)
1036 if (!req->first_trb)
1037 req->first_trb = &ep->transfer_ring[ep->enq_ptr];
1039 for (i = 0; i < count; i++) {
1040 struct tegra_xudc_trb *trb = &ep->transfer_ring[ep->enq_ptr];
1043 if ((i == count - 1) || (wait_td && i == count - 2))
1046 tegra_xudc_queue_one_trb(ep, req, trb, ioc);
1047 req->last_trb = trb;
1050 if (ep->enq_ptr == XUDC_TRANSFER_RING_SIZE - 1) {
1051 trb = &ep->transfer_ring[ep->enq_ptr];
1052 trb_write_cycle(trb, ep->pcs);
1064 static void tegra_xudc_ep_ring_doorbell(struct tegra_xudc_ep *ep)
1066 struct tegra_xudc *xudc = ep->xudc;
1069 if (list_empty(&ep->queue))
1072 val = DB_TARGET(ep->index);
1073 if (usb_endpoint_xfer_control(ep->desc)) {
1074 val |= DB_STREAMID(xudc->setup_seq_num);
1075 } else if (usb_ss_max_streams(ep->comp_desc) > 0) {
1076 struct tegra_xudc_request *req;
1078 /* Don't ring doorbell if the stream has been rejected. */
1079 if (ep->stream_rejected)
1082 req = list_first_entry(&ep->queue, struct tegra_xudc_request,
1084 val |= DB_STREAMID(req->usb_req.stream_id);
1087 dev_dbg(xudc->dev, "ring doorbell: %#x\n", val);
1088 xudc_writel(xudc, val, DB);
1091 static void tegra_xudc_ep_kick_queue(struct tegra_xudc_ep *ep)
1093 struct tegra_xudc_request *req;
1094 bool trbs_queued = false;
1096 list_for_each_entry(req, &ep->queue, list) {
1100 if (tegra_xudc_queue_trbs(ep, req) > 0)
1105 tegra_xudc_ep_ring_doorbell(ep);
1109 __tegra_xudc_ep_queue(struct tegra_xudc_ep *ep, struct tegra_xudc_request *req)
1111 struct tegra_xudc *xudc = ep->xudc;
1114 if (usb_endpoint_xfer_control(ep->desc) && !list_empty(&ep->queue)) {
1115 dev_err(xudc->dev, "control EP has pending transfers\n");
1119 if (usb_endpoint_xfer_control(ep->desc)) {
1120 err = usb_gadget_map_request(&xudc->gadget, &req->usb_req,
1121 (xudc->setup_state ==
1124 err = usb_gadget_map_request(&xudc->gadget, &req->usb_req,
1125 usb_endpoint_dir_in(ep->desc));
1129 dev_err(xudc->dev, "failed to map request: %d\n", err);
1133 req->first_trb = NULL;
1134 req->last_trb = NULL;
1135 req->buf_queued = 0;
1136 req->trbs_queued = 0;
1137 req->need_zlp = false;
1138 req->trbs_needed = DIV_ROUND_UP(req->usb_req.length,
1139 XUDC_TRB_MAX_BUFFER_SIZE);
1140 if (req->usb_req.length == 0)
1143 if (!usb_endpoint_xfer_isoc(ep->desc) &&
1144 req->usb_req.zero && req->usb_req.length &&
1145 ((req->usb_req.length % ep->usb_ep.maxpacket) == 0)) {
1147 req->need_zlp = true;
1150 req->usb_req.status = -EINPROGRESS;
1151 req->usb_req.actual = 0;
1153 list_add_tail(&req->list, &ep->queue);
1155 tegra_xudc_ep_kick_queue(ep);
1161 tegra_xudc_ep_queue(struct usb_ep *usb_ep, struct usb_request *usb_req,
1164 struct tegra_xudc_request *req;
1165 struct tegra_xudc_ep *ep;
1166 struct tegra_xudc *xudc;
1167 unsigned long flags;
1170 if (!usb_ep || !usb_req)
1173 ep = to_xudc_ep(usb_ep);
1174 req = to_xudc_req(usb_req);
1177 spin_lock_irqsave(&xudc->lock, flags);
1178 if (xudc->powergated || !ep->desc) {
1183 ret = __tegra_xudc_ep_queue(ep, req);
1185 spin_unlock_irqrestore(&xudc->lock, flags);
1190 static void squeeze_transfer_ring(struct tegra_xudc_ep *ep,
1191 struct tegra_xudc_request *req)
1193 struct tegra_xudc_trb *trb = req->first_trb;
1194 bool pcs_enq = trb_read_cycle(trb);
1198 * Clear out all the TRBs part of or after the cancelled request,
1199 * and must correct trb cycle bit to the last un-enqueued state.
1201 while (trb != &ep->transfer_ring[ep->enq_ptr]) {
1202 pcs = trb_read_cycle(trb);
1203 memset(trb, 0, sizeof(*trb));
1204 trb_write_cycle(trb, !pcs);
1207 if (trb_read_type(trb) == TRB_TYPE_LINK)
1208 trb = ep->transfer_ring;
1211 /* Requests will be re-queued at the start of the cancelled request. */
1212 ep->enq_ptr = req->first_trb - ep->transfer_ring;
1214 * Retrieve the correct cycle bit state from the first trb of
1215 * the cancelled request.
1218 ep->ring_full = false;
1219 list_for_each_entry_continue(req, &ep->queue, list) {
1220 req->usb_req.status = -EINPROGRESS;
1221 req->usb_req.actual = 0;
1223 req->first_trb = NULL;
1224 req->last_trb = NULL;
1225 req->buf_queued = 0;
1226 req->trbs_queued = 0;
1231 * Determine if the given TRB is in the range [first trb, last trb] for the
1234 static bool trb_in_request(struct tegra_xudc_ep *ep,
1235 struct tegra_xudc_request *req,
1236 struct tegra_xudc_trb *trb)
1238 dev_dbg(ep->xudc->dev, "%s: request %p -> %p; trb %p\n", __func__,
1239 req->first_trb, req->last_trb, trb);
1241 if (trb >= req->first_trb && (trb <= req->last_trb ||
1242 req->last_trb < req->first_trb))
1245 if (trb < req->first_trb && trb <= req->last_trb &&
1246 req->last_trb < req->first_trb)
1253 * Determine if the given TRB is in the range [EP enqueue pointer, first TRB)
1254 * for the given endpoint and request.
1256 static bool trb_before_request(struct tegra_xudc_ep *ep,
1257 struct tegra_xudc_request *req,
1258 struct tegra_xudc_trb *trb)
1260 struct tegra_xudc_trb *enq_trb = &ep->transfer_ring[ep->enq_ptr];
1262 dev_dbg(ep->xudc->dev, "%s: request %p -> %p; enq ptr: %p; trb %p\n",
1263 __func__, req->first_trb, req->last_trb, enq_trb, trb);
1265 if (trb < req->first_trb && (enq_trb <= trb ||
1266 req->first_trb < enq_trb))
1269 if (trb > req->first_trb && req->first_trb < enq_trb && enq_trb <= trb)
1276 __tegra_xudc_ep_dequeue(struct tegra_xudc_ep *ep,
1277 struct tegra_xudc_request *req)
1279 struct tegra_xudc *xudc = ep->xudc;
1280 struct tegra_xudc_request *r;
1281 struct tegra_xudc_trb *deq_trb;
1282 bool busy, kick_queue = false;
1285 /* Make sure the request is actually queued to this endpoint. */
1286 list_for_each_entry(r, &ep->queue, list) {
1294 /* Request hasn't been queued in the transfer ring yet. */
1295 if (!req->trbs_queued) {
1296 tegra_xudc_req_done(ep, req, -ECONNRESET);
1300 /* Halt DMA for this endpiont. */
1301 if (ep_ctx_read_state(ep->context) == EP_STATE_RUNNING) {
1302 ep_pause(xudc, ep->index);
1303 ep_wait_for_inactive(xudc, ep->index);
1306 deq_trb = trb_phys_to_virt(ep, ep_ctx_read_deq_ptr(ep->context));
1307 /* Is the hardware processing the TRB at the dequeue pointer? */
1308 busy = (trb_read_cycle(deq_trb) == ep_ctx_read_dcs(ep->context));
1310 if (trb_in_request(ep, req, deq_trb) && busy) {
1312 * Request has been partially completed or it hasn't
1313 * started processing yet.
1317 squeeze_transfer_ring(ep, req);
1319 req->usb_req.actual = ep_ctx_read_edtla(ep->context);
1320 tegra_xudc_req_done(ep, req, -ECONNRESET);
1323 /* EDTLA is > 0: request has been partially completed */
1324 if (req->usb_req.actual > 0) {
1326 * Abort the pending transfer and update the dequeue
1329 ep_ctx_write_edtla(ep->context, 0);
1330 ep_ctx_write_partial_td(ep->context, 0);
1331 ep_ctx_write_data_offset(ep->context, 0);
1333 deq_ptr = trb_virt_to_phys(ep,
1334 &ep->transfer_ring[ep->enq_ptr]);
1336 if (dma_mapping_error(xudc->dev, deq_ptr)) {
1339 ep_ctx_write_deq_ptr(ep->context, deq_ptr);
1340 ep_ctx_write_dcs(ep->context, ep->pcs);
1341 ep_reload(xudc, ep->index);
1344 } else if (trb_before_request(ep, req, deq_trb) && busy) {
1345 /* Request hasn't started processing yet. */
1346 squeeze_transfer_ring(ep, req);
1348 tegra_xudc_req_done(ep, req, -ECONNRESET);
1352 * Request has completed, but we haven't processed the
1353 * completion event yet.
1355 tegra_xudc_req_done(ep, req, -ECONNRESET);
1359 /* Resume the endpoint. */
1360 ep_unpause(xudc, ep->index);
1363 tegra_xudc_ep_kick_queue(ep);
1369 tegra_xudc_ep_dequeue(struct usb_ep *usb_ep, struct usb_request *usb_req)
1371 struct tegra_xudc_request *req;
1372 struct tegra_xudc_ep *ep;
1373 struct tegra_xudc *xudc;
1374 unsigned long flags;
1377 if (!usb_ep || !usb_req)
1380 ep = to_xudc_ep(usb_ep);
1381 req = to_xudc_req(usb_req);
1384 spin_lock_irqsave(&xudc->lock, flags);
1386 if (xudc->powergated || !ep->desc) {
1391 ret = __tegra_xudc_ep_dequeue(ep, req);
1393 spin_unlock_irqrestore(&xudc->lock, flags);
1398 static int __tegra_xudc_ep_set_halt(struct tegra_xudc_ep *ep, bool halt)
1400 struct tegra_xudc *xudc = ep->xudc;
1405 if (usb_endpoint_xfer_isoc(ep->desc)) {
1406 dev_err(xudc->dev, "can't halt isoc EP\n");
1410 if (!!(xudc_readl(xudc, EP_HALT) & BIT(ep->index)) == halt) {
1411 dev_dbg(xudc->dev, "EP %u already %s\n", ep->index,
1412 halt ? "halted" : "not halted");
1417 ep_halt(xudc, ep->index);
1419 ep_ctx_write_state(ep->context, EP_STATE_DISABLED);
1421 ep_reload(xudc, ep->index);
1423 ep_ctx_write_state(ep->context, EP_STATE_RUNNING);
1424 ep_ctx_write_seq_num(ep->context, 0);
1426 ep_reload(xudc, ep->index);
1427 ep_unpause(xudc, ep->index);
1428 ep_unhalt(xudc, ep->index);
1430 tegra_xudc_ep_ring_doorbell(ep);
1436 static int tegra_xudc_ep_set_halt(struct usb_ep *usb_ep, int value)
1438 struct tegra_xudc_ep *ep;
1439 struct tegra_xudc *xudc;
1440 unsigned long flags;
1446 ep = to_xudc_ep(usb_ep);
1449 spin_lock_irqsave(&xudc->lock, flags);
1450 if (xudc->powergated) {
1455 if (value && usb_endpoint_dir_in(ep->desc) &&
1456 !list_empty(&ep->queue)) {
1457 dev_err(xudc->dev, "can't halt EP with requests pending\n");
1462 ret = __tegra_xudc_ep_set_halt(ep, value);
1464 spin_unlock_irqrestore(&xudc->lock, flags);
1469 static void tegra_xudc_ep_context_setup(struct tegra_xudc_ep *ep)
1471 const struct usb_endpoint_descriptor *desc = ep->desc;
1472 const struct usb_ss_ep_comp_descriptor *comp_desc = ep->comp_desc;
1473 struct tegra_xudc *xudc = ep->xudc;
1474 u16 maxpacket, maxburst = 0, esit = 0;
1477 maxpacket = usb_endpoint_maxp(desc) & 0x7ff;
1478 if (xudc->gadget.speed == USB_SPEED_SUPER) {
1479 if (!usb_endpoint_xfer_control(desc))
1480 maxburst = comp_desc->bMaxBurst;
1482 if (usb_endpoint_xfer_int(desc) || usb_endpoint_xfer_isoc(desc))
1483 esit = le16_to_cpu(comp_desc->wBytesPerInterval);
1484 } else if ((xudc->gadget.speed < USB_SPEED_SUPER) &&
1485 (usb_endpoint_xfer_int(desc) ||
1486 usb_endpoint_xfer_isoc(desc))) {
1487 if (xudc->gadget.speed == USB_SPEED_HIGH) {
1488 maxburst = (usb_endpoint_maxp(desc) >> 11) & 0x3;
1489 if (maxburst == 0x3) {
1491 "invalid endpoint maxburst\n");
1495 esit = maxpacket * (maxburst + 1);
1498 memset(ep->context, 0, sizeof(*ep->context));
1500 ep_ctx_write_state(ep->context, EP_STATE_RUNNING);
1501 ep_ctx_write_interval(ep->context, desc->bInterval);
1502 if (xudc->gadget.speed == USB_SPEED_SUPER) {
1503 if (usb_endpoint_xfer_isoc(desc)) {
1504 ep_ctx_write_mult(ep->context,
1505 comp_desc->bmAttributes & 0x3);
1508 if (usb_endpoint_xfer_bulk(desc)) {
1509 ep_ctx_write_max_pstreams(ep->context,
1510 comp_desc->bmAttributes &
1512 ep_ctx_write_lsa(ep->context, 1);
1516 if (!usb_endpoint_xfer_control(desc) && usb_endpoint_dir_out(desc))
1517 val = usb_endpoint_type(desc);
1519 val = usb_endpoint_type(desc) + EP_TYPE_CONTROL;
1521 ep_ctx_write_type(ep->context, val);
1522 ep_ctx_write_cerr(ep->context, 0x3);
1523 ep_ctx_write_max_packet_size(ep->context, maxpacket);
1524 ep_ctx_write_max_burst_size(ep->context, maxburst);
1526 ep_ctx_write_deq_ptr(ep->context, ep->transfer_ring_phys);
1527 ep_ctx_write_dcs(ep->context, ep->pcs);
1529 /* Select a reasonable average TRB length based on endpoint type. */
1530 switch (usb_endpoint_type(desc)) {
1531 case USB_ENDPOINT_XFER_CONTROL:
1534 case USB_ENDPOINT_XFER_INT:
1537 case USB_ENDPOINT_XFER_BULK:
1538 case USB_ENDPOINT_XFER_ISOC:
1544 ep_ctx_write_avg_trb_len(ep->context, val);
1545 ep_ctx_write_max_esit_payload(ep->context, esit);
1547 ep_ctx_write_cerrcnt(ep->context, 0x3);
1550 static void setup_link_trb(struct tegra_xudc_ep *ep,
1551 struct tegra_xudc_trb *trb)
1553 trb_write_data_ptr(trb, ep->transfer_ring_phys);
1554 trb_write_type(trb, TRB_TYPE_LINK);
1555 trb_write_toggle_cycle(trb, 1);
1558 static int __tegra_xudc_ep_disable(struct tegra_xudc_ep *ep)
1560 struct tegra_xudc *xudc = ep->xudc;
1562 if (ep_ctx_read_state(ep->context) == EP_STATE_DISABLED) {
1563 dev_err(xudc->dev, "endpoint %u already disabled\n",
1568 ep_ctx_write_state(ep->context, EP_STATE_DISABLED);
1570 ep_reload(xudc, ep->index);
1572 tegra_xudc_ep_nuke(ep, -ESHUTDOWN);
1574 xudc->nr_enabled_eps--;
1575 if (usb_endpoint_xfer_isoc(ep->desc))
1576 xudc->nr_isoch_eps--;
1579 ep->comp_desc = NULL;
1581 memset(ep->context, 0, sizeof(*ep->context));
1583 ep_unpause(xudc, ep->index);
1584 ep_unhalt(xudc, ep->index);
1585 if (xudc_readl(xudc, EP_STOPPED) & BIT(ep->index))
1586 xudc_writel(xudc, BIT(ep->index), EP_STOPPED);
1589 * If this is the last endpoint disabled in a de-configure request,
1590 * switch back to address state.
1592 if ((xudc->device_state == USB_STATE_CONFIGURED) &&
1593 (xudc->nr_enabled_eps == 1)) {
1596 xudc->device_state = USB_STATE_ADDRESS;
1597 usb_gadget_set_state(&xudc->gadget, xudc->device_state);
1599 val = xudc_readl(xudc, CTRL);
1601 xudc_writel(xudc, val, CTRL);
1604 dev_info(xudc->dev, "ep %u disabled\n", ep->index);
1609 static int tegra_xudc_ep_disable(struct usb_ep *usb_ep)
1611 struct tegra_xudc_ep *ep;
1612 struct tegra_xudc *xudc;
1613 unsigned long flags;
1619 ep = to_xudc_ep(usb_ep);
1622 spin_lock_irqsave(&xudc->lock, flags);
1623 if (xudc->powergated) {
1628 ret = __tegra_xudc_ep_disable(ep);
1630 spin_unlock_irqrestore(&xudc->lock, flags);
1635 static int __tegra_xudc_ep_enable(struct tegra_xudc_ep *ep,
1636 const struct usb_endpoint_descriptor *desc)
1638 struct tegra_xudc *xudc = ep->xudc;
1642 if (xudc->gadget.speed == USB_SPEED_SUPER &&
1643 !usb_endpoint_xfer_control(desc) && !ep->usb_ep.comp_desc)
1646 /* Disable the EP if it is not disabled */
1647 if (ep_ctx_read_state(ep->context) != EP_STATE_DISABLED)
1648 __tegra_xudc_ep_disable(ep);
1651 ep->comp_desc = ep->usb_ep.comp_desc;
1653 if (usb_endpoint_xfer_isoc(desc)) {
1654 if (xudc->nr_isoch_eps > XUDC_MAX_ISOCH_EPS) {
1655 dev_err(xudc->dev, "too many isoch endpoints\n");
1658 xudc->nr_isoch_eps++;
1661 memset(ep->transfer_ring, 0, XUDC_TRANSFER_RING_SIZE *
1662 sizeof(*ep->transfer_ring));
1663 setup_link_trb(ep, &ep->transfer_ring[XUDC_TRANSFER_RING_SIZE - 1]);
1668 ep->ring_full = false;
1669 xudc->nr_enabled_eps++;
1671 tegra_xudc_ep_context_setup(ep);
1674 * No need to reload and un-halt EP0. This will be done automatically
1675 * once a valid SETUP packet is received.
1677 if (usb_endpoint_xfer_control(desc))
1681 * Transition to configured state once the first non-control
1682 * endpoint is enabled.
1684 if (xudc->device_state == USB_STATE_ADDRESS) {
1685 val = xudc_readl(xudc, CTRL);
1687 xudc_writel(xudc, val, CTRL);
1689 xudc->device_state = USB_STATE_CONFIGURED;
1690 usb_gadget_set_state(&xudc->gadget, xudc->device_state);
1693 if (usb_endpoint_xfer_isoc(desc)) {
1695 * Pause all bulk endpoints when enabling an isoch endpoint
1696 * to ensure the isoch endpoint is allocated enough bandwidth.
1698 for (i = 0; i < ARRAY_SIZE(xudc->ep); i++) {
1699 if (xudc->ep[i].desc &&
1700 usb_endpoint_xfer_bulk(xudc->ep[i].desc))
1705 ep_reload(xudc, ep->index);
1706 ep_unpause(xudc, ep->index);
1707 ep_unhalt(xudc, ep->index);
1709 if (usb_endpoint_xfer_isoc(desc)) {
1710 for (i = 0; i < ARRAY_SIZE(xudc->ep); i++) {
1711 if (xudc->ep[i].desc &&
1712 usb_endpoint_xfer_bulk(xudc->ep[i].desc))
1713 ep_unpause(xudc, i);
1718 dev_info(xudc->dev, "EP %u (type: %s, dir: %s) enabled\n", ep->index,
1719 usb_ep_type_string(usb_endpoint_type(ep->desc)),
1720 usb_endpoint_dir_in(ep->desc) ? "in" : "out");
1725 static int tegra_xudc_ep_enable(struct usb_ep *usb_ep,
1726 const struct usb_endpoint_descriptor *desc)
1728 struct tegra_xudc_ep *ep;
1729 struct tegra_xudc *xudc;
1730 unsigned long flags;
1733 if (!usb_ep || !desc || (desc->bDescriptorType != USB_DT_ENDPOINT))
1736 ep = to_xudc_ep(usb_ep);
1739 spin_lock_irqsave(&xudc->lock, flags);
1740 if (xudc->powergated) {
1745 ret = __tegra_xudc_ep_enable(ep, desc);
1747 spin_unlock_irqrestore(&xudc->lock, flags);
1752 static struct usb_request *
1753 tegra_xudc_ep_alloc_request(struct usb_ep *usb_ep, gfp_t gfp)
1755 struct tegra_xudc_request *req;
1757 req = kzalloc(sizeof(*req), gfp);
1761 INIT_LIST_HEAD(&req->list);
1763 return &req->usb_req;
1766 static void tegra_xudc_ep_free_request(struct usb_ep *usb_ep,
1767 struct usb_request *usb_req)
1769 struct tegra_xudc_request *req = to_xudc_req(usb_req);
1774 static struct usb_ep_ops tegra_xudc_ep_ops = {
1775 .enable = tegra_xudc_ep_enable,
1776 .disable = tegra_xudc_ep_disable,
1777 .alloc_request = tegra_xudc_ep_alloc_request,
1778 .free_request = tegra_xudc_ep_free_request,
1779 .queue = tegra_xudc_ep_queue,
1780 .dequeue = tegra_xudc_ep_dequeue,
1781 .set_halt = tegra_xudc_ep_set_halt,
1784 static int tegra_xudc_ep0_enable(struct usb_ep *usb_ep,
1785 const struct usb_endpoint_descriptor *desc)
1790 static int tegra_xudc_ep0_disable(struct usb_ep *usb_ep)
1795 static struct usb_ep_ops tegra_xudc_ep0_ops = {
1796 .enable = tegra_xudc_ep0_enable,
1797 .disable = tegra_xudc_ep0_disable,
1798 .alloc_request = tegra_xudc_ep_alloc_request,
1799 .free_request = tegra_xudc_ep_free_request,
1800 .queue = tegra_xudc_ep_queue,
1801 .dequeue = tegra_xudc_ep_dequeue,
1802 .set_halt = tegra_xudc_ep_set_halt,
1805 static int tegra_xudc_gadget_get_frame(struct usb_gadget *gadget)
1807 struct tegra_xudc *xudc = to_xudc(gadget);
1808 unsigned long flags;
1811 spin_lock_irqsave(&xudc->lock, flags);
1812 if (xudc->powergated) {
1817 ret = (xudc_readl(xudc, MFINDEX) & MFINDEX_FRAME_MASK) >>
1818 MFINDEX_FRAME_SHIFT;
1820 spin_unlock_irqrestore(&xudc->lock, flags);
1825 static void tegra_xudc_resume_device_state(struct tegra_xudc *xudc)
1830 ep_unpause_all(xudc);
1832 /* Direct link to U0. */
1833 val = xudc_readl(xudc, PORTSC);
1834 if (((val & PORTSC_PLS_MASK) >> PORTSC_PLS_SHIFT) != PORTSC_PLS_U0) {
1835 val &= ~(PORTSC_CHANGE_MASK | PORTSC_PLS_MASK);
1836 val |= PORTSC_LWS | PORTSC_PLS(PORTSC_PLS_U0);
1837 xudc_writel(xudc, val, PORTSC);
1840 if (xudc->device_state == USB_STATE_SUSPENDED) {
1841 xudc->device_state = xudc->resume_state;
1842 usb_gadget_set_state(&xudc->gadget, xudc->device_state);
1843 xudc->resume_state = 0;
1847 * Doorbells may be dropped if they are sent too soon (< ~200ns)
1848 * after unpausing the endpoint. Wait for 500ns just to be safe.
1851 for (i = 0; i < ARRAY_SIZE(xudc->ep); i++)
1852 tegra_xudc_ep_ring_doorbell(&xudc->ep[i]);
1855 static int tegra_xudc_gadget_wakeup(struct usb_gadget *gadget)
1857 struct tegra_xudc *xudc = to_xudc(gadget);
1858 unsigned long flags;
1862 spin_lock_irqsave(&xudc->lock, flags);
1864 if (xudc->powergated) {
1868 val = xudc_readl(xudc, PORTPM);
1869 dev_dbg(xudc->dev, "%s: PORTPM=%#x, speed=%x\n", __func__,
1870 val, gadget->speed);
1872 if (((xudc->gadget.speed <= USB_SPEED_HIGH) &&
1873 (val & PORTPM_RWE)) ||
1874 ((xudc->gadget.speed == USB_SPEED_SUPER) &&
1875 (val & PORTPM_FRWE))) {
1876 tegra_xudc_resume_device_state(xudc);
1878 /* Send Device Notification packet. */
1879 if (xudc->gadget.speed == USB_SPEED_SUPER) {
1880 val = DEVNOTIF_LO_TYPE(DEVNOTIF_LO_TYPE_FUNCTION_WAKE)
1882 xudc_writel(xudc, 0, DEVNOTIF_HI);
1883 xudc_writel(xudc, val, DEVNOTIF_LO);
1888 dev_dbg(xudc->dev, "%s: ret value is %d", __func__, ret);
1889 spin_unlock_irqrestore(&xudc->lock, flags);
1894 static int tegra_xudc_gadget_pullup(struct usb_gadget *gadget, int is_on)
1896 struct tegra_xudc *xudc = to_xudc(gadget);
1897 unsigned long flags;
1900 pm_runtime_get_sync(xudc->dev);
1902 spin_lock_irqsave(&xudc->lock, flags);
1904 if (is_on != xudc->pullup) {
1905 val = xudc_readl(xudc, CTRL);
1909 val &= ~CTRL_ENABLE;
1910 xudc_writel(xudc, val, CTRL);
1913 xudc->pullup = is_on;
1914 dev_dbg(xudc->dev, "%s: pullup:%d", __func__, is_on);
1916 spin_unlock_irqrestore(&xudc->lock, flags);
1918 pm_runtime_put(xudc->dev);
1923 static int tegra_xudc_gadget_start(struct usb_gadget *gadget,
1924 struct usb_gadget_driver *driver)
1926 struct tegra_xudc *xudc = to_xudc(gadget);
1927 unsigned long flags;
1934 pm_runtime_get_sync(xudc->dev);
1936 spin_lock_irqsave(&xudc->lock, flags);
1943 xudc->setup_state = WAIT_FOR_SETUP;
1944 xudc->device_state = USB_STATE_DEFAULT;
1945 usb_gadget_set_state(&xudc->gadget, xudc->device_state);
1947 ret = __tegra_xudc_ep_enable(&xudc->ep[0], &tegra_xudc_ep0_desc);
1951 val = xudc_readl(xudc, CTRL);
1952 val |= CTRL_IE | CTRL_LSE;
1953 xudc_writel(xudc, val, CTRL);
1955 val = xudc_readl(xudc, PORTHALT);
1956 val |= PORTHALT_STCHG_INTR_EN;
1957 xudc_writel(xudc, val, PORTHALT);
1960 val = xudc_readl(xudc, CTRL);
1962 xudc_writel(xudc, val, CTRL);
1965 xudc->driver = driver;
1967 dev_dbg(xudc->dev, "%s: ret value is %d", __func__, ret);
1968 spin_unlock_irqrestore(&xudc->lock, flags);
1970 pm_runtime_put(xudc->dev);
1975 static int tegra_xudc_gadget_stop(struct usb_gadget *gadget)
1977 struct tegra_xudc *xudc = to_xudc(gadget);
1978 unsigned long flags;
1981 pm_runtime_get_sync(xudc->dev);
1983 spin_lock_irqsave(&xudc->lock, flags);
1985 val = xudc_readl(xudc, CTRL);
1986 val &= ~(CTRL_IE | CTRL_ENABLE);
1987 xudc_writel(xudc, val, CTRL);
1989 __tegra_xudc_ep_disable(&xudc->ep[0]);
1991 xudc->driver = NULL;
1992 dev_dbg(xudc->dev, "Gadget stopped");
1994 spin_unlock_irqrestore(&xudc->lock, flags);
1996 pm_runtime_put(xudc->dev);
2001 static int tegra_xudc_set_selfpowered(struct usb_gadget *gadget, int is_on)
2003 struct tegra_xudc *xudc = to_xudc(gadget);
2005 dev_dbg(xudc->dev, "%s: %d\n", __func__, is_on);
2006 xudc->selfpowered = !!is_on;
2011 static struct usb_gadget_ops tegra_xudc_gadget_ops = {
2012 .get_frame = tegra_xudc_gadget_get_frame,
2013 .wakeup = tegra_xudc_gadget_wakeup,
2014 .pullup = tegra_xudc_gadget_pullup,
2015 .udc_start = tegra_xudc_gadget_start,
2016 .udc_stop = tegra_xudc_gadget_stop,
2017 .set_selfpowered = tegra_xudc_set_selfpowered,
2020 static void no_op_complete(struct usb_ep *ep, struct usb_request *req)
2025 tegra_xudc_ep0_queue_status(struct tegra_xudc *xudc,
2026 void (*cmpl)(struct usb_ep *, struct usb_request *))
2028 xudc->ep0_req->usb_req.buf = NULL;
2029 xudc->ep0_req->usb_req.dma = 0;
2030 xudc->ep0_req->usb_req.length = 0;
2031 xudc->ep0_req->usb_req.complete = cmpl;
2032 xudc->ep0_req->usb_req.context = xudc;
2034 return __tegra_xudc_ep_queue(&xudc->ep[0], xudc->ep0_req);
2038 tegra_xudc_ep0_queue_data(struct tegra_xudc *xudc, void *buf, size_t len,
2039 void (*cmpl)(struct usb_ep *, struct usb_request *))
2041 xudc->ep0_req->usb_req.buf = buf;
2042 xudc->ep0_req->usb_req.length = len;
2043 xudc->ep0_req->usb_req.complete = cmpl;
2044 xudc->ep0_req->usb_req.context = xudc;
2046 return __tegra_xudc_ep_queue(&xudc->ep[0], xudc->ep0_req);
2049 static void tegra_xudc_ep0_req_done(struct tegra_xudc *xudc)
2051 switch (xudc->setup_state) {
2052 case DATA_STAGE_XFER:
2053 xudc->setup_state = STATUS_STAGE_RECV;
2054 tegra_xudc_ep0_queue_status(xudc, no_op_complete);
2056 case DATA_STAGE_RECV:
2057 xudc->setup_state = STATUS_STAGE_XFER;
2058 tegra_xudc_ep0_queue_status(xudc, no_op_complete);
2061 xudc->setup_state = WAIT_FOR_SETUP;
2066 static int tegra_xudc_ep0_delegate_req(struct tegra_xudc *xudc,
2067 struct usb_ctrlrequest *ctrl)
2071 spin_unlock(&xudc->lock);
2072 ret = xudc->driver->setup(&xudc->gadget, ctrl);
2073 spin_lock(&xudc->lock);
2078 static void set_feature_complete(struct usb_ep *ep, struct usb_request *req)
2080 struct tegra_xudc *xudc = req->context;
2082 if (xudc->test_mode_pattern) {
2083 xudc_writel(xudc, xudc->test_mode_pattern, PORT_TM);
2084 xudc->test_mode_pattern = 0;
2088 static int tegra_xudc_ep0_set_feature(struct tegra_xudc *xudc,
2089 struct usb_ctrlrequest *ctrl)
2091 bool set = (ctrl->bRequest == USB_REQ_SET_FEATURE);
2092 u32 feature = le16_to_cpu(ctrl->wValue);
2093 u32 index = le16_to_cpu(ctrl->wIndex);
2097 if (le16_to_cpu(ctrl->wLength) != 0)
2100 switch (ctrl->bRequestType & USB_RECIP_MASK) {
2101 case USB_RECIP_DEVICE:
2103 case USB_DEVICE_REMOTE_WAKEUP:
2104 if ((xudc->gadget.speed == USB_SPEED_SUPER) ||
2105 (xudc->device_state == USB_STATE_DEFAULT))
2108 val = xudc_readl(xudc, PORTPM);
2114 xudc_writel(xudc, val, PORTPM);
2116 case USB_DEVICE_U1_ENABLE:
2117 case USB_DEVICE_U2_ENABLE:
2118 if ((xudc->device_state != USB_STATE_CONFIGURED) ||
2119 (xudc->gadget.speed != USB_SPEED_SUPER))
2122 val = xudc_readl(xudc, PORTPM);
2123 if ((feature == USB_DEVICE_U1_ENABLE) &&
2124 xudc->soc->u1_enable) {
2131 if ((feature == USB_DEVICE_U2_ENABLE) &&
2132 xudc->soc->u2_enable) {
2139 xudc_writel(xudc, val, PORTPM);
2141 case USB_DEVICE_TEST_MODE:
2142 if (xudc->gadget.speed != USB_SPEED_HIGH)
2148 xudc->test_mode_pattern = index >> 8;
2155 case USB_RECIP_INTERFACE:
2156 if (xudc->device_state != USB_STATE_CONFIGURED)
2160 case USB_INTRF_FUNC_SUSPEND:
2162 val = xudc_readl(xudc, PORTPM);
2164 if (index & USB_INTRF_FUNC_SUSPEND_RW)
2167 val &= ~PORTPM_FRWE;
2169 xudc_writel(xudc, val, PORTPM);
2172 return tegra_xudc_ep0_delegate_req(xudc, ctrl);
2178 case USB_RECIP_ENDPOINT:
2179 ep = (index & USB_ENDPOINT_NUMBER_MASK) * 2 +
2180 ((index & USB_DIR_IN) ? 1 : 0);
2182 if ((xudc->device_state == USB_STATE_DEFAULT) ||
2183 ((xudc->device_state == USB_STATE_ADDRESS) &&
2187 ret = __tegra_xudc_ep_set_halt(&xudc->ep[ep], set);
2195 return tegra_xudc_ep0_queue_status(xudc, set_feature_complete);
2198 static int tegra_xudc_ep0_get_status(struct tegra_xudc *xudc,
2199 struct usb_ctrlrequest *ctrl)
2201 struct tegra_xudc_ep_context *ep_ctx;
2202 u32 val, ep, index = le16_to_cpu(ctrl->wIndex);
2205 if (!(ctrl->bRequestType & USB_DIR_IN))
2208 if ((le16_to_cpu(ctrl->wValue) != 0) ||
2209 (le16_to_cpu(ctrl->wLength) != 2))
2212 switch (ctrl->bRequestType & USB_RECIP_MASK) {
2213 case USB_RECIP_DEVICE:
2214 val = xudc_readl(xudc, PORTPM);
2216 if (xudc->selfpowered)
2217 status |= BIT(USB_DEVICE_SELF_POWERED);
2219 if ((xudc->gadget.speed < USB_SPEED_SUPER) &&
2221 status |= BIT(USB_DEVICE_REMOTE_WAKEUP);
2223 if (xudc->gadget.speed == USB_SPEED_SUPER) {
2224 if (val & PORTPM_U1E)
2225 status |= BIT(USB_DEV_STAT_U1_ENABLED);
2226 if (val & PORTPM_U2E)
2227 status |= BIT(USB_DEV_STAT_U2_ENABLED);
2230 case USB_RECIP_INTERFACE:
2231 if (xudc->gadget.speed == USB_SPEED_SUPER) {
2232 status |= USB_INTRF_STAT_FUNC_RW_CAP;
2233 val = xudc_readl(xudc, PORTPM);
2234 if (val & PORTPM_FRWE)
2235 status |= USB_INTRF_STAT_FUNC_RW;
2238 case USB_RECIP_ENDPOINT:
2239 ep = (index & USB_ENDPOINT_NUMBER_MASK) * 2 +
2240 ((index & USB_DIR_IN) ? 1 : 0);
2241 ep_ctx = &xudc->ep_context[ep];
2243 if ((xudc->device_state != USB_STATE_CONFIGURED) &&
2244 ((xudc->device_state != USB_STATE_ADDRESS) || (ep != 0)))
2247 if (ep_ctx_read_state(ep_ctx) == EP_STATE_DISABLED)
2250 if (xudc_readl(xudc, EP_HALT) & BIT(ep))
2251 status |= BIT(USB_ENDPOINT_HALT);
2257 xudc->status_buf = cpu_to_le16(status);
2258 return tegra_xudc_ep0_queue_data(xudc, &xudc->status_buf,
2259 sizeof(xudc->status_buf),
2263 static void set_sel_complete(struct usb_ep *ep, struct usb_request *req)
2265 /* Nothing to do with SEL values */
2268 static int tegra_xudc_ep0_set_sel(struct tegra_xudc *xudc,
2269 struct usb_ctrlrequest *ctrl)
2271 if (ctrl->bRequestType != (USB_DIR_OUT | USB_RECIP_DEVICE |
2275 if (xudc->device_state == USB_STATE_DEFAULT)
2278 if ((le16_to_cpu(ctrl->wIndex) != 0) ||
2279 (le16_to_cpu(ctrl->wValue) != 0) ||
2280 (le16_to_cpu(ctrl->wLength) != 6))
2283 return tegra_xudc_ep0_queue_data(xudc, &xudc->sel_timing,
2284 sizeof(xudc->sel_timing),
2288 static void set_isoch_delay_complete(struct usb_ep *ep, struct usb_request *req)
2290 /* Nothing to do with isoch delay */
2293 static int tegra_xudc_ep0_set_isoch_delay(struct tegra_xudc *xudc,
2294 struct usb_ctrlrequest *ctrl)
2296 u32 delay = le16_to_cpu(ctrl->wValue);
2298 if (ctrl->bRequestType != (USB_DIR_OUT | USB_RECIP_DEVICE |
2302 if ((delay > 65535) || (le16_to_cpu(ctrl->wIndex) != 0) ||
2303 (le16_to_cpu(ctrl->wLength) != 0))
2306 xudc->isoch_delay = delay;
2308 return tegra_xudc_ep0_queue_status(xudc, set_isoch_delay_complete);
2311 static void set_address_complete(struct usb_ep *ep, struct usb_request *req)
2313 struct tegra_xudc *xudc = req->context;
2315 if ((xudc->device_state == USB_STATE_DEFAULT) &&
2316 (xudc->dev_addr != 0)) {
2317 xudc->device_state = USB_STATE_ADDRESS;
2318 usb_gadget_set_state(&xudc->gadget, xudc->device_state);
2319 } else if ((xudc->device_state == USB_STATE_ADDRESS) &&
2320 (xudc->dev_addr == 0)) {
2321 xudc->device_state = USB_STATE_DEFAULT;
2322 usb_gadget_set_state(&xudc->gadget, xudc->device_state);
2326 static int tegra_xudc_ep0_set_address(struct tegra_xudc *xudc,
2327 struct usb_ctrlrequest *ctrl)
2329 struct tegra_xudc_ep *ep0 = &xudc->ep[0];
2330 u32 val, addr = le16_to_cpu(ctrl->wValue);
2332 if (ctrl->bRequestType != (USB_DIR_OUT | USB_RECIP_DEVICE |
2336 if ((addr > 127) || (le16_to_cpu(ctrl->wIndex) != 0) ||
2337 (le16_to_cpu(ctrl->wLength) != 0))
2340 if (xudc->device_state == USB_STATE_CONFIGURED)
2343 dev_dbg(xudc->dev, "set address: %u\n", addr);
2345 xudc->dev_addr = addr;
2346 val = xudc_readl(xudc, CTRL);
2347 val &= ~(CTRL_DEVADDR_MASK);
2348 val |= CTRL_DEVADDR(addr);
2349 xudc_writel(xudc, val, CTRL);
2351 ep_ctx_write_devaddr(ep0->context, addr);
2353 return tegra_xudc_ep0_queue_status(xudc, set_address_complete);
2356 static int tegra_xudc_ep0_standard_req(struct tegra_xudc *xudc,
2357 struct usb_ctrlrequest *ctrl)
2361 switch (ctrl->bRequest) {
2362 case USB_REQ_GET_STATUS:
2363 dev_dbg(xudc->dev, "USB_REQ_GET_STATUS\n");
2364 ret = tegra_xudc_ep0_get_status(xudc, ctrl);
2366 case USB_REQ_SET_ADDRESS:
2367 dev_dbg(xudc->dev, "USB_REQ_SET_ADDRESS\n");
2368 ret = tegra_xudc_ep0_set_address(xudc, ctrl);
2370 case USB_REQ_SET_SEL:
2371 dev_dbg(xudc->dev, "USB_REQ_SET_SEL\n");
2372 ret = tegra_xudc_ep0_set_sel(xudc, ctrl);
2374 case USB_REQ_SET_ISOCH_DELAY:
2375 dev_dbg(xudc->dev, "USB_REQ_SET_ISOCH_DELAY\n");
2376 ret = tegra_xudc_ep0_set_isoch_delay(xudc, ctrl);
2378 case USB_REQ_CLEAR_FEATURE:
2379 case USB_REQ_SET_FEATURE:
2380 dev_dbg(xudc->dev, "USB_REQ_CLEAR/SET_FEATURE\n");
2381 ret = tegra_xudc_ep0_set_feature(xudc, ctrl);
2383 case USB_REQ_SET_CONFIGURATION:
2384 dev_dbg(xudc->dev, "USB_REQ_SET_CONFIGURATION\n");
2386 * In theory we need to clear RUN bit before status stage of
2387 * deconfig request sent, but this seems to be causing problems.
2388 * Clear RUN once all endpoints are disabled instead.
2392 ret = tegra_xudc_ep0_delegate_req(xudc, ctrl);
2399 static void tegra_xudc_handle_ep0_setup_packet(struct tegra_xudc *xudc,
2400 struct usb_ctrlrequest *ctrl,
2405 xudc->setup_seq_num = seq_num;
2407 /* Ensure EP0 is unhalted. */
2411 * On Tegra210, setup packets with sequence numbers 0xfffe or 0xffff
2412 * are invalid. Halt EP0 until we get a valid packet.
2414 if (xudc->soc->invalid_seq_num &&
2415 (seq_num == 0xfffe || seq_num == 0xffff)) {
2416 dev_warn(xudc->dev, "invalid sequence number detected\n");
2422 xudc->setup_state = (ctrl->bRequestType & USB_DIR_IN) ?
2423 DATA_STAGE_XFER : DATA_STAGE_RECV;
2425 xudc->setup_state = STATUS_STAGE_XFER;
2427 if ((ctrl->bRequestType & USB_TYPE_MASK) == USB_TYPE_STANDARD)
2428 ret = tegra_xudc_ep0_standard_req(xudc, ctrl);
2430 ret = tegra_xudc_ep0_delegate_req(xudc, ctrl);
2433 dev_warn(xudc->dev, "setup request failed: %d\n", ret);
2434 xudc->setup_state = WAIT_FOR_SETUP;
2439 static void tegra_xudc_handle_ep0_event(struct tegra_xudc *xudc,
2440 struct tegra_xudc_trb *event)
2442 struct usb_ctrlrequest *ctrl = (struct usb_ctrlrequest *)event;
2443 u16 seq_num = trb_read_seq_num(event);
2445 if (xudc->setup_state != WAIT_FOR_SETUP) {
2447 * The controller is in the process of handling another
2448 * setup request. Queue subsequent requests and handle
2449 * the last one once the controller reports a sequence
2452 memcpy(&xudc->setup_packet.ctrl_req, ctrl, sizeof(*ctrl));
2453 xudc->setup_packet.seq_num = seq_num;
2454 xudc->queued_setup_packet = true;
2456 tegra_xudc_handle_ep0_setup_packet(xudc, ctrl, seq_num);
2460 static struct tegra_xudc_request *
2461 trb_to_request(struct tegra_xudc_ep *ep, struct tegra_xudc_trb *trb)
2463 struct tegra_xudc_request *req;
2465 list_for_each_entry(req, &ep->queue, list) {
2466 if (!req->trbs_queued)
2469 if (trb_in_request(ep, req, trb))
2476 static void tegra_xudc_handle_transfer_completion(struct tegra_xudc *xudc,
2477 struct tegra_xudc_ep *ep,
2478 struct tegra_xudc_trb *event)
2480 struct tegra_xudc_request *req;
2481 struct tegra_xudc_trb *trb;
2484 short_packet = (trb_read_cmpl_code(event) ==
2485 TRB_CMPL_CODE_SHORT_PACKET);
2487 trb = trb_phys_to_virt(ep, trb_read_data_ptr(event));
2488 req = trb_to_request(ep, trb);
2491 * TDs are complete on short packet or when the completed TRB is the
2492 * last TRB in the TD (the CHAIN bit is unset).
2494 if (req && (short_packet || (!trb_read_chain(trb) &&
2495 (req->trbs_needed == req->trbs_queued)))) {
2496 struct tegra_xudc_trb *last = req->last_trb;
2497 unsigned int residual;
2499 residual = trb_read_transfer_len(event);
2500 req->usb_req.actual = req->usb_req.length - residual;
2502 dev_dbg(xudc->dev, "bytes transferred %u / %u\n",
2503 req->usb_req.actual, req->usb_req.length);
2505 tegra_xudc_req_done(ep, req, 0);
2507 if (ep->desc && usb_endpoint_xfer_control(ep->desc))
2508 tegra_xudc_ep0_req_done(xudc);
2511 * Advance the dequeue pointer past the end of the current TD
2512 * on short packet completion.
2515 ep->deq_ptr = (last - ep->transfer_ring) + 1;
2516 if (ep->deq_ptr == XUDC_TRANSFER_RING_SIZE - 1)
2520 dev_warn(xudc->dev, "transfer event on dequeued request\n");
2524 tegra_xudc_ep_kick_queue(ep);
2527 static void tegra_xudc_handle_transfer_event(struct tegra_xudc *xudc,
2528 struct tegra_xudc_trb *event)
2530 unsigned int ep_index = trb_read_endpoint_id(event);
2531 struct tegra_xudc_ep *ep = &xudc->ep[ep_index];
2532 struct tegra_xudc_trb *trb;
2535 if (ep_ctx_read_state(ep->context) == EP_STATE_DISABLED) {
2536 dev_warn(xudc->dev, "transfer event on disabled EP %u\n",
2541 /* Update transfer ring dequeue pointer. */
2542 trb = trb_phys_to_virt(ep, trb_read_data_ptr(event));
2543 comp_code = trb_read_cmpl_code(event);
2544 if (comp_code != TRB_CMPL_CODE_BABBLE_DETECTED_ERR) {
2545 ep->deq_ptr = (trb - ep->transfer_ring) + 1;
2547 if (ep->deq_ptr == XUDC_TRANSFER_RING_SIZE - 1)
2549 ep->ring_full = false;
2552 switch (comp_code) {
2553 case TRB_CMPL_CODE_SUCCESS:
2554 case TRB_CMPL_CODE_SHORT_PACKET:
2555 tegra_xudc_handle_transfer_completion(xudc, ep, event);
2557 case TRB_CMPL_CODE_HOST_REJECTED:
2558 dev_info(xudc->dev, "stream rejected on EP %u\n", ep_index);
2560 ep->stream_rejected = true;
2562 case TRB_CMPL_CODE_PRIME_PIPE_RECEIVED:
2563 dev_info(xudc->dev, "prime pipe received on EP %u\n", ep_index);
2565 if (ep->stream_rejected) {
2566 ep->stream_rejected = false;
2568 * An EP is stopped when a stream is rejected. Wait
2569 * for the EP to report that it is stopped and then
2572 ep_wait_for_stopped(xudc, ep_index);
2574 tegra_xudc_ep_ring_doorbell(ep);
2576 case TRB_CMPL_CODE_BABBLE_DETECTED_ERR:
2578 * Wait for the EP to be stopped so the controller stops
2579 * processing doorbells.
2581 ep_wait_for_stopped(xudc, ep_index);
2582 ep->enq_ptr = ep->deq_ptr;
2583 tegra_xudc_ep_nuke(ep, -EIO);
2585 case TRB_CMPL_CODE_STREAM_NUMP_ERROR:
2586 case TRB_CMPL_CODE_CTRL_DIR_ERR:
2587 case TRB_CMPL_CODE_INVALID_STREAM_TYPE_ERR:
2588 case TRB_CMPL_CODE_RING_UNDERRUN:
2589 case TRB_CMPL_CODE_RING_OVERRUN:
2590 case TRB_CMPL_CODE_ISOCH_BUFFER_OVERRUN:
2591 case TRB_CMPL_CODE_USB_TRANS_ERR:
2592 case TRB_CMPL_CODE_TRB_ERR:
2593 dev_err(xudc->dev, "completion error %#x on EP %u\n",
2594 comp_code, ep_index);
2596 ep_halt(xudc, ep_index);
2598 case TRB_CMPL_CODE_CTRL_SEQNUM_ERR:
2599 dev_info(xudc->dev, "sequence number error\n");
2602 * Kill any queued control request and skip to the last
2603 * setup packet we received.
2605 tegra_xudc_ep_nuke(ep, -EINVAL);
2606 xudc->setup_state = WAIT_FOR_SETUP;
2607 if (!xudc->queued_setup_packet)
2610 tegra_xudc_handle_ep0_setup_packet(xudc,
2611 &xudc->setup_packet.ctrl_req,
2612 xudc->setup_packet.seq_num);
2613 xudc->queued_setup_packet = false;
2615 case TRB_CMPL_CODE_STOPPED:
2616 dev_dbg(xudc->dev, "stop completion code on EP %u\n",
2620 tegra_xudc_ep_nuke(ep, -ECONNREFUSED);
2623 dev_dbg(xudc->dev, "completion event %#x on EP %u\n",
2624 comp_code, ep_index);
2629 static void tegra_xudc_reset(struct tegra_xudc *xudc)
2631 struct tegra_xudc_ep *ep0 = &xudc->ep[0];
2635 xudc->setup_state = WAIT_FOR_SETUP;
2636 xudc->device_state = USB_STATE_DEFAULT;
2637 usb_gadget_set_state(&xudc->gadget, xudc->device_state);
2639 ep_unpause_all(xudc);
2641 for (i = 0; i < ARRAY_SIZE(xudc->ep); i++)
2642 tegra_xudc_ep_nuke(&xudc->ep[i], -ESHUTDOWN);
2645 * Reset sequence number and dequeue pointer to flush the transfer
2648 ep0->deq_ptr = ep0->enq_ptr;
2649 ep0->ring_full = false;
2651 xudc->setup_seq_num = 0;
2652 xudc->queued_setup_packet = false;
2654 ep_ctx_write_seq_num(ep0->context, xudc->setup_seq_num);
2656 deq_ptr = trb_virt_to_phys(ep0, &ep0->transfer_ring[ep0->deq_ptr]);
2658 if (!dma_mapping_error(xudc->dev, deq_ptr)) {
2659 ep_ctx_write_deq_ptr(ep0->context, deq_ptr);
2660 ep_ctx_write_dcs(ep0->context, ep0->pcs);
2663 ep_unhalt_all(xudc);
2665 ep_unpause(xudc, 0);
2668 static void tegra_xudc_port_connect(struct tegra_xudc *xudc)
2670 struct tegra_xudc_ep *ep0 = &xudc->ep[0];
2674 val = (xudc_readl(xudc, PORTSC) & PORTSC_PS_MASK) >> PORTSC_PS_SHIFT;
2677 xudc->gadget.speed = USB_SPEED_LOW;
2680 xudc->gadget.speed = USB_SPEED_FULL;
2683 xudc->gadget.speed = USB_SPEED_HIGH;
2686 xudc->gadget.speed = USB_SPEED_SUPER;
2689 xudc->gadget.speed = USB_SPEED_UNKNOWN;
2693 xudc->device_state = USB_STATE_DEFAULT;
2694 usb_gadget_set_state(&xudc->gadget, xudc->device_state);
2696 xudc->setup_state = WAIT_FOR_SETUP;
2698 if (xudc->gadget.speed == USB_SPEED_SUPER)
2703 ep_ctx_write_max_packet_size(ep0->context, maxpacket);
2704 tegra_xudc_ep0_desc.wMaxPacketSize = cpu_to_le16(maxpacket);
2705 usb_ep_set_maxpacket_limit(&ep0->usb_ep, maxpacket);
2707 if (!xudc->soc->u1_enable) {
2708 val = xudc_readl(xudc, PORTPM);
2709 val &= ~(PORTPM_U1TIMEOUT_MASK);
2710 xudc_writel(xudc, val, PORTPM);
2713 if (!xudc->soc->u2_enable) {
2714 val = xudc_readl(xudc, PORTPM);
2715 val &= ~(PORTPM_U2TIMEOUT_MASK);
2716 xudc_writel(xudc, val, PORTPM);
2719 if (xudc->gadget.speed <= USB_SPEED_HIGH) {
2720 val = xudc_readl(xudc, PORTPM);
2721 val &= ~(PORTPM_L1S_MASK);
2722 if (xudc->soc->lpm_enable)
2723 val |= PORTPM_L1S(PORTPM_L1S_ACCEPT);
2725 val |= PORTPM_L1S(PORTPM_L1S_NYET);
2726 xudc_writel(xudc, val, PORTPM);
2729 val = xudc_readl(xudc, ST);
2731 xudc_writel(xudc, ST_RC, ST);
2734 static void tegra_xudc_port_disconnect(struct tegra_xudc *xudc)
2736 tegra_xudc_reset(xudc);
2738 if (xudc->driver && xudc->driver->disconnect) {
2739 spin_unlock(&xudc->lock);
2740 xudc->driver->disconnect(&xudc->gadget);
2741 spin_lock(&xudc->lock);
2744 xudc->device_state = USB_STATE_NOTATTACHED;
2745 usb_gadget_set_state(&xudc->gadget, xudc->device_state);
2747 complete(&xudc->disconnect_complete);
2750 static void tegra_xudc_port_reset(struct tegra_xudc *xudc)
2752 tegra_xudc_reset(xudc);
2755 spin_unlock(&xudc->lock);
2756 usb_gadget_udc_reset(&xudc->gadget, xudc->driver);
2757 spin_lock(&xudc->lock);
2760 tegra_xudc_port_connect(xudc);
2763 static void tegra_xudc_port_suspend(struct tegra_xudc *xudc)
2765 dev_dbg(xudc->dev, "port suspend\n");
2767 xudc->resume_state = xudc->device_state;
2768 xudc->device_state = USB_STATE_SUSPENDED;
2769 usb_gadget_set_state(&xudc->gadget, xudc->device_state);
2771 if (xudc->driver->suspend) {
2772 spin_unlock(&xudc->lock);
2773 xudc->driver->suspend(&xudc->gadget);
2774 spin_lock(&xudc->lock);
2778 static void tegra_xudc_port_resume(struct tegra_xudc *xudc)
2780 dev_dbg(xudc->dev, "port resume\n");
2782 tegra_xudc_resume_device_state(xudc);
2784 if (xudc->driver->resume) {
2785 spin_unlock(&xudc->lock);
2786 xudc->driver->resume(&xudc->gadget);
2787 spin_lock(&xudc->lock);
2791 static inline void clear_port_change(struct tegra_xudc *xudc, u32 flag)
2795 val = xudc_readl(xudc, PORTSC);
2796 val &= ~PORTSC_CHANGE_MASK;
2798 xudc_writel(xudc, val, PORTSC);
2801 static void __tegra_xudc_handle_port_status(struct tegra_xudc *xudc)
2803 u32 portsc, porthalt;
2805 porthalt = xudc_readl(xudc, PORTHALT);
2806 if ((porthalt & PORTHALT_STCHG_REQ) &&
2807 (porthalt & PORTHALT_HALT_LTSSM)) {
2808 dev_dbg(xudc->dev, "STCHG_REQ, PORTHALT = %#x\n", porthalt);
2809 porthalt &= ~PORTHALT_HALT_LTSSM;
2810 xudc_writel(xudc, porthalt, PORTHALT);
2813 portsc = xudc_readl(xudc, PORTSC);
2814 if ((portsc & PORTSC_PRC) && (portsc & PORTSC_PR)) {
2815 dev_dbg(xudc->dev, "PRC, PR, PORTSC = %#x\n", portsc);
2816 clear_port_change(xudc, PORTSC_PRC | PORTSC_PED);
2817 #define TOGGLE_VBUS_WAIT_MS 100
2818 if (xudc->soc->port_reset_quirk) {
2819 schedule_delayed_work(&xudc->port_reset_war_work,
2820 msecs_to_jiffies(TOGGLE_VBUS_WAIT_MS));
2821 xudc->wait_for_sec_prc = 1;
2825 if ((portsc & PORTSC_PRC) && !(portsc & PORTSC_PR)) {
2826 dev_dbg(xudc->dev, "PRC, Not PR, PORTSC = %#x\n", portsc);
2827 clear_port_change(xudc, PORTSC_PRC | PORTSC_PED);
2828 tegra_xudc_port_reset(xudc);
2829 cancel_delayed_work(&xudc->port_reset_war_work);
2830 xudc->wait_for_sec_prc = 0;
2833 portsc = xudc_readl(xudc, PORTSC);
2834 if (portsc & PORTSC_WRC) {
2835 dev_dbg(xudc->dev, "WRC, PORTSC = %#x\n", portsc);
2836 clear_port_change(xudc, PORTSC_WRC | PORTSC_PED);
2837 if (!(xudc_readl(xudc, PORTSC) & PORTSC_WPR))
2838 tegra_xudc_port_reset(xudc);
2841 portsc = xudc_readl(xudc, PORTSC);
2842 if (portsc & PORTSC_CSC) {
2843 dev_dbg(xudc->dev, "CSC, PORTSC = %#x\n", portsc);
2844 clear_port_change(xudc, PORTSC_CSC);
2846 if (portsc & PORTSC_CCS)
2847 tegra_xudc_port_connect(xudc);
2849 tegra_xudc_port_disconnect(xudc);
2851 if (xudc->wait_csc) {
2852 cancel_delayed_work(&xudc->plc_reset_work);
2853 xudc->wait_csc = false;
2857 portsc = xudc_readl(xudc, PORTSC);
2858 if (portsc & PORTSC_PLC) {
2859 u32 pls = (portsc & PORTSC_PLS_MASK) >> PORTSC_PLS_SHIFT;
2861 dev_dbg(xudc->dev, "PLC, PORTSC = %#x\n", portsc);
2862 clear_port_change(xudc, PORTSC_PLC);
2865 tegra_xudc_port_suspend(xudc);
2868 if (xudc->gadget.speed < USB_SPEED_SUPER)
2869 tegra_xudc_port_resume(xudc);
2871 case PORTSC_PLS_RESUME:
2872 if (xudc->gadget.speed == USB_SPEED_SUPER)
2873 tegra_xudc_port_resume(xudc);
2875 case PORTSC_PLS_INACTIVE:
2876 schedule_delayed_work(&xudc->plc_reset_work,
2877 msecs_to_jiffies(TOGGLE_VBUS_WAIT_MS));
2878 xudc->wait_csc = true;
2885 if (portsc & PORTSC_CEC) {
2886 dev_warn(xudc->dev, "CEC, PORTSC = %#x\n", portsc);
2887 clear_port_change(xudc, PORTSC_CEC);
2890 dev_dbg(xudc->dev, "PORTSC = %#x\n", xudc_readl(xudc, PORTSC));
2893 static void tegra_xudc_handle_port_status(struct tegra_xudc *xudc)
2895 while ((xudc_readl(xudc, PORTSC) & PORTSC_CHANGE_MASK) ||
2896 (xudc_readl(xudc, PORTHALT) & PORTHALT_STCHG_REQ))
2897 __tegra_xudc_handle_port_status(xudc);
2900 static void tegra_xudc_handle_event(struct tegra_xudc *xudc,
2901 struct tegra_xudc_trb *event)
2903 u32 type = trb_read_type(event);
2905 dump_trb(xudc, "EVENT", event);
2908 case TRB_TYPE_PORT_STATUS_CHANGE_EVENT:
2909 tegra_xudc_handle_port_status(xudc);
2911 case TRB_TYPE_TRANSFER_EVENT:
2912 tegra_xudc_handle_transfer_event(xudc, event);
2914 case TRB_TYPE_SETUP_PACKET_EVENT:
2915 tegra_xudc_handle_ep0_event(xudc, event);
2918 dev_info(xudc->dev, "Unrecognized TRB type = %#x\n", type);
2923 static void tegra_xudc_process_event_ring(struct tegra_xudc *xudc)
2925 struct tegra_xudc_trb *event;
2929 event = xudc->event_ring[xudc->event_ring_index] +
2930 xudc->event_ring_deq_ptr;
2932 if (trb_read_cycle(event) != xudc->ccs)
2935 tegra_xudc_handle_event(xudc, event);
2937 xudc->event_ring_deq_ptr++;
2938 if (xudc->event_ring_deq_ptr == XUDC_EVENT_RING_SIZE) {
2939 xudc->event_ring_deq_ptr = 0;
2940 xudc->event_ring_index++;
2943 if (xudc->event_ring_index == XUDC_NR_EVENT_RINGS) {
2944 xudc->event_ring_index = 0;
2945 xudc->ccs = !xudc->ccs;
2949 erdp = xudc->event_ring_phys[xudc->event_ring_index] +
2950 xudc->event_ring_deq_ptr * sizeof(*event);
2952 xudc_writel(xudc, upper_32_bits(erdp), ERDPHI);
2953 xudc_writel(xudc, lower_32_bits(erdp) | ERDPLO_EHB, ERDPLO);
2956 static irqreturn_t tegra_xudc_irq(int irq, void *data)
2958 struct tegra_xudc *xudc = data;
2959 unsigned long flags;
2962 val = xudc_readl(xudc, ST);
2965 xudc_writel(xudc, ST_IP, ST);
2967 spin_lock_irqsave(&xudc->lock, flags);
2968 tegra_xudc_process_event_ring(xudc);
2969 spin_unlock_irqrestore(&xudc->lock, flags);
2974 static int tegra_xudc_alloc_ep(struct tegra_xudc *xudc, unsigned int index)
2976 struct tegra_xudc_ep *ep = &xudc->ep[index];
2980 ep->context = &xudc->ep_context[index];
2981 INIT_LIST_HEAD(&ep->queue);
2984 * EP1 would be the input endpoint corresponding to EP0, but since
2985 * EP0 is bi-directional, EP1 is unused.
2990 ep->transfer_ring = dma_pool_alloc(xudc->transfer_ring_pool,
2992 &ep->transfer_ring_phys);
2993 if (!ep->transfer_ring)
2997 snprintf(ep->name, sizeof(ep->name), "ep%u%s", index / 2,
2998 (index % 2 == 0) ? "out" : "in");
2999 ep->usb_ep.name = ep->name;
3000 usb_ep_set_maxpacket_limit(&ep->usb_ep, 1024);
3001 ep->usb_ep.max_streams = 16;
3002 ep->usb_ep.ops = &tegra_xudc_ep_ops;
3003 ep->usb_ep.caps.type_bulk = true;
3004 ep->usb_ep.caps.type_int = true;
3006 ep->usb_ep.caps.dir_in = true;
3008 ep->usb_ep.caps.dir_out = true;
3009 list_add_tail(&ep->usb_ep.ep_list, &xudc->gadget.ep_list);
3011 strscpy(ep->name, "ep0", 3);
3012 ep->usb_ep.name = ep->name;
3013 usb_ep_set_maxpacket_limit(&ep->usb_ep, 512);
3014 ep->usb_ep.ops = &tegra_xudc_ep0_ops;
3015 ep->usb_ep.caps.type_control = true;
3016 ep->usb_ep.caps.dir_in = true;
3017 ep->usb_ep.caps.dir_out = true;
3023 static void tegra_xudc_free_ep(struct tegra_xudc *xudc, unsigned int index)
3025 struct tegra_xudc_ep *ep = &xudc->ep[index];
3028 * EP1 would be the input endpoint corresponding to EP0, but since
3029 * EP0 is bi-directional, EP1 is unused.
3034 dma_pool_free(xudc->transfer_ring_pool, ep->transfer_ring,
3035 ep->transfer_ring_phys);
3038 static int tegra_xudc_alloc_eps(struct tegra_xudc *xudc)
3040 struct usb_request *req;
3045 dma_alloc_coherent(xudc->dev, XUDC_NR_EPS *
3046 sizeof(*xudc->ep_context),
3047 &xudc->ep_context_phys, GFP_KERNEL);
3048 if (!xudc->ep_context)
3051 xudc->transfer_ring_pool =
3052 dmam_pool_create(dev_name(xudc->dev), xudc->dev,
3053 XUDC_TRANSFER_RING_SIZE *
3054 sizeof(struct tegra_xudc_trb),
3055 sizeof(struct tegra_xudc_trb), 0);
3056 if (!xudc->transfer_ring_pool) {
3058 goto free_ep_context;
3061 INIT_LIST_HEAD(&xudc->gadget.ep_list);
3062 for (i = 0; i < ARRAY_SIZE(xudc->ep); i++) {
3063 err = tegra_xudc_alloc_ep(xudc, i);
3068 req = tegra_xudc_ep_alloc_request(&xudc->ep[0].usb_ep, GFP_KERNEL);
3073 xudc->ep0_req = to_xudc_req(req);
3079 tegra_xudc_free_ep(xudc, i - 1);
3081 dma_free_coherent(xudc->dev, XUDC_NR_EPS * sizeof(*xudc->ep_context),
3082 xudc->ep_context, xudc->ep_context_phys);
3086 static void tegra_xudc_init_eps(struct tegra_xudc *xudc)
3088 xudc_writel(xudc, lower_32_bits(xudc->ep_context_phys), ECPLO);
3089 xudc_writel(xudc, upper_32_bits(xudc->ep_context_phys), ECPHI);
3092 static void tegra_xudc_free_eps(struct tegra_xudc *xudc)
3096 tegra_xudc_ep_free_request(&xudc->ep[0].usb_ep,
3097 &xudc->ep0_req->usb_req);
3099 for (i = 0; i < ARRAY_SIZE(xudc->ep); i++)
3100 tegra_xudc_free_ep(xudc, i);
3102 dma_free_coherent(xudc->dev, XUDC_NR_EPS * sizeof(*xudc->ep_context),
3103 xudc->ep_context, xudc->ep_context_phys);
3106 static int tegra_xudc_alloc_event_ring(struct tegra_xudc *xudc)
3110 for (i = 0; i < ARRAY_SIZE(xudc->event_ring); i++) {
3111 xudc->event_ring[i] =
3112 dma_alloc_coherent(xudc->dev, XUDC_EVENT_RING_SIZE *
3113 sizeof(*xudc->event_ring[i]),
3114 &xudc->event_ring_phys[i],
3116 if (!xudc->event_ring[i])
3123 for (; i > 0; i--) {
3124 dma_free_coherent(xudc->dev, XUDC_EVENT_RING_SIZE *
3125 sizeof(*xudc->event_ring[i - 1]),
3126 xudc->event_ring[i - 1],
3127 xudc->event_ring_phys[i - 1]);
3132 static void tegra_xudc_init_event_ring(struct tegra_xudc *xudc)
3137 val = xudc_readl(xudc, SPARAM);
3138 val &= ~(SPARAM_ERSTMAX_MASK);
3139 val |= SPARAM_ERSTMAX(XUDC_NR_EVENT_RINGS);
3140 xudc_writel(xudc, val, SPARAM);
3142 for (i = 0; i < ARRAY_SIZE(xudc->event_ring); i++) {
3143 memset(xudc->event_ring[i], 0, XUDC_EVENT_RING_SIZE *
3144 sizeof(*xudc->event_ring[i]));
3146 val = xudc_readl(xudc, ERSTSZ);
3147 val &= ~(ERSTSZ_ERSTXSZ_MASK << ERSTSZ_ERSTXSZ_SHIFT(i));
3148 val |= XUDC_EVENT_RING_SIZE << ERSTSZ_ERSTXSZ_SHIFT(i);
3149 xudc_writel(xudc, val, ERSTSZ);
3151 xudc_writel(xudc, lower_32_bits(xudc->event_ring_phys[i]),
3153 xudc_writel(xudc, upper_32_bits(xudc->event_ring_phys[i]),
3157 val = lower_32_bits(xudc->event_ring_phys[0]);
3158 xudc_writel(xudc, val, ERDPLO);
3160 xudc_writel(xudc, val, EREPLO);
3162 val = upper_32_bits(xudc->event_ring_phys[0]);
3163 xudc_writel(xudc, val, ERDPHI);
3164 xudc_writel(xudc, val, EREPHI);
3167 xudc->event_ring_index = 0;
3168 xudc->event_ring_deq_ptr = 0;
3171 static void tegra_xudc_free_event_ring(struct tegra_xudc *xudc)
3175 for (i = 0; i < ARRAY_SIZE(xudc->event_ring); i++) {
3176 dma_free_coherent(xudc->dev, XUDC_EVENT_RING_SIZE *
3177 sizeof(*xudc->event_ring[i]),
3178 xudc->event_ring[i],
3179 xudc->event_ring_phys[i]);
3183 static void tegra_xudc_fpci_ipfs_init(struct tegra_xudc *xudc)
3187 if (xudc->soc->has_ipfs) {
3188 val = ipfs_readl(xudc, XUSB_DEV_CONFIGURATION_0);
3189 val |= XUSB_DEV_CONFIGURATION_0_EN_FPCI;
3190 ipfs_writel(xudc, val, XUSB_DEV_CONFIGURATION_0);
3191 usleep_range(10, 15);
3194 /* Enable bus master */
3195 val = XUSB_DEV_CFG_1_IO_SPACE_EN | XUSB_DEV_CFG_1_MEMORY_SPACE_EN |
3196 XUSB_DEV_CFG_1_BUS_MASTER_EN;
3197 fpci_writel(xudc, val, XUSB_DEV_CFG_1);
3199 /* Program BAR0 space */
3200 val = fpci_readl(xudc, XUSB_DEV_CFG_4);
3201 val &= ~(XUSB_DEV_CFG_4_BASE_ADDR_MASK);
3202 val |= xudc->phys_base & (XUSB_DEV_CFG_4_BASE_ADDR_MASK);
3204 fpci_writel(xudc, val, XUSB_DEV_CFG_4);
3205 fpci_writel(xudc, upper_32_bits(xudc->phys_base), XUSB_DEV_CFG_5);
3207 usleep_range(100, 200);
3209 if (xudc->soc->has_ipfs) {
3210 /* Enable interrupt assertion */
3211 val = ipfs_readl(xudc, XUSB_DEV_INTR_MASK_0);
3212 val |= XUSB_DEV_INTR_MASK_0_IP_INT_MASK;
3213 ipfs_writel(xudc, val, XUSB_DEV_INTR_MASK_0);
3217 static void tegra_xudc_device_params_init(struct tegra_xudc *xudc)
3221 if (xudc->soc->has_ipfs) {
3222 val = xudc_readl(xudc, BLCG);
3224 val &= ~(BLCG_DFPCI | BLCG_UFPCI | BLCG_FE |
3225 BLCG_COREPLL_PWRDN);
3226 val |= BLCG_IOPLL_0_PWRDN;
3227 val |= BLCG_IOPLL_1_PWRDN;
3228 val |= BLCG_IOPLL_2_PWRDN;
3230 xudc_writel(xudc, val, BLCG);
3233 /* Set a reasonable U3 exit timer value. */
3234 val = xudc_readl(xudc, SSPX_CORE_PADCTL4);
3235 val &= ~(SSPX_CORE_PADCTL4_RXDAT_VLD_TIMEOUT_U3_MASK);
3236 val |= SSPX_CORE_PADCTL4_RXDAT_VLD_TIMEOUT_U3(0x5dc0);
3237 xudc_writel(xudc, val, SSPX_CORE_PADCTL4);
3239 /* Default ping LFPS tBurst is too large. */
3240 val = xudc_readl(xudc, SSPX_CORE_CNT0);
3241 val &= ~(SSPX_CORE_CNT0_PING_TBURST_MASK);
3242 val |= SSPX_CORE_CNT0_PING_TBURST(0xa);
3243 xudc_writel(xudc, val, SSPX_CORE_CNT0);
3245 /* Default tPortConfiguration timeout is too small. */
3246 val = xudc_readl(xudc, SSPX_CORE_CNT30);
3247 val &= ~(SSPX_CORE_CNT30_LMPITP_TIMER_MASK);
3248 val |= SSPX_CORE_CNT30_LMPITP_TIMER(0x978);
3249 xudc_writel(xudc, val, SSPX_CORE_CNT30);
3251 if (xudc->soc->lpm_enable) {
3252 /* Set L1 resume duration to 95 us. */
3253 val = xudc_readl(xudc, HSFSPI_COUNT13);
3254 val &= ~(HSFSPI_COUNT13_U2_RESUME_K_DURATION_MASK);
3255 val |= HSFSPI_COUNT13_U2_RESUME_K_DURATION(0x2c88);
3256 xudc_writel(xudc, val, HSFSPI_COUNT13);
3260 * Compliacne suite appears to be violating polling LFPS tBurst max
3261 * of 1.4us. Send 1.45us instead.
3263 val = xudc_readl(xudc, SSPX_CORE_CNT32);
3264 val &= ~(SSPX_CORE_CNT32_POLL_TBURST_MAX_MASK);
3265 val |= SSPX_CORE_CNT32_POLL_TBURST_MAX(0xb0);
3266 xudc_writel(xudc, val, SSPX_CORE_CNT32);
3268 /* Direct HS/FS port instance to RxDetect. */
3269 val = xudc_readl(xudc, CFG_DEV_FE);
3270 val &= ~(CFG_DEV_FE_PORTREGSEL_MASK);
3271 val |= CFG_DEV_FE_PORTREGSEL(CFG_DEV_FE_PORTREGSEL_HSFS_PI);
3272 xudc_writel(xudc, val, CFG_DEV_FE);
3274 val = xudc_readl(xudc, PORTSC);
3275 val &= ~(PORTSC_CHANGE_MASK | PORTSC_PLS_MASK);
3276 val |= PORTSC_LWS | PORTSC_PLS(PORTSC_PLS_RXDETECT);
3277 xudc_writel(xudc, val, PORTSC);
3279 /* Direct SS port instance to RxDetect. */
3280 val = xudc_readl(xudc, CFG_DEV_FE);
3281 val &= ~(CFG_DEV_FE_PORTREGSEL_MASK);
3282 val |= CFG_DEV_FE_PORTREGSEL_SS_PI & CFG_DEV_FE_PORTREGSEL_MASK;
3283 xudc_writel(xudc, val, CFG_DEV_FE);
3285 val = xudc_readl(xudc, PORTSC);
3286 val &= ~(PORTSC_CHANGE_MASK | PORTSC_PLS_MASK);
3287 val |= PORTSC_LWS | PORTSC_PLS(PORTSC_PLS_RXDETECT);
3288 xudc_writel(xudc, val, PORTSC);
3290 /* Restore port instance. */
3291 val = xudc_readl(xudc, CFG_DEV_FE);
3292 val &= ~(CFG_DEV_FE_PORTREGSEL_MASK);
3293 xudc_writel(xudc, val, CFG_DEV_FE);
3296 * Enable INFINITE_SS_RETRY to prevent device from entering
3297 * Disabled.Error when attached to buggy SuperSpeed hubs.
3299 val = xudc_readl(xudc, CFG_DEV_FE);
3300 val |= CFG_DEV_FE_INFINITE_SS_RETRY;
3301 xudc_writel(xudc, val, CFG_DEV_FE);
3303 /* Set interrupt moderation. */
3304 imod = XUDC_INTERRUPT_MODERATION_US * 4;
3305 val = xudc_readl(xudc, RT_IMOD);
3306 val &= ~((RT_IMOD_IMODI_MASK) | (RT_IMOD_IMODC_MASK));
3307 val |= (RT_IMOD_IMODI(imod) | RT_IMOD_IMODC(imod));
3308 xudc_writel(xudc, val, RT_IMOD);
3310 /* increase SSPI transaction timeout from 32us to 512us */
3311 val = xudc_readl(xudc, CFG_DEV_SSPI_XFER);
3312 val &= ~(CFG_DEV_SSPI_XFER_ACKTIMEOUT_MASK);
3313 val |= CFG_DEV_SSPI_XFER_ACKTIMEOUT(0xf000);
3314 xudc_writel(xudc, val, CFG_DEV_SSPI_XFER);
3317 static int tegra_xudc_phy_init(struct tegra_xudc *xudc)
3321 err = phy_init(xudc->utmi_phy);
3323 dev_err(xudc->dev, "utmi phy init failed: %d\n", err);
3327 err = phy_init(xudc->usb3_phy);
3329 dev_err(xudc->dev, "usb3 phy init failed: %d\n", err);
3336 phy_exit(xudc->utmi_phy);
3340 static void tegra_xudc_phy_exit(struct tegra_xudc *xudc)
3342 phy_exit(xudc->usb3_phy);
3343 phy_exit(xudc->utmi_phy);
3346 static const char * const tegra210_xudc_supply_names[] = {
3351 static const char * const tegra210_xudc_clock_names[] = {
3359 static const char * const tegra186_xudc_clock_names[] = {
3366 static struct tegra_xudc_soc tegra210_xudc_soc_data = {
3367 .supply_names = tegra210_xudc_supply_names,
3368 .num_supplies = ARRAY_SIZE(tegra210_xudc_supply_names),
3369 .clock_names = tegra210_xudc_clock_names,
3370 .num_clks = ARRAY_SIZE(tegra210_xudc_clock_names),
3373 .lpm_enable = false,
3374 .invalid_seq_num = true,
3376 .port_reset_quirk = true,
3380 static struct tegra_xudc_soc tegra186_xudc_soc_data = {
3381 .clock_names = tegra186_xudc_clock_names,
3382 .num_clks = ARRAY_SIZE(tegra186_xudc_clock_names),
3385 .lpm_enable = false,
3386 .invalid_seq_num = false,
3388 .port_reset_quirk = false,
3392 static const struct of_device_id tegra_xudc_of_match[] = {
3394 .compatible = "nvidia,tegra210-xudc",
3395 .data = &tegra210_xudc_soc_data
3398 .compatible = "nvidia,tegra186-xudc",
3399 .data = &tegra186_xudc_soc_data
3403 MODULE_DEVICE_TABLE(of, tegra_xudc_of_match);
3405 static void tegra_xudc_powerdomain_remove(struct tegra_xudc *xudc)
3407 if (xudc->genpd_dl_ss)
3408 device_link_del(xudc->genpd_dl_ss);
3409 if (xudc->genpd_dl_device)
3410 device_link_del(xudc->genpd_dl_device);
3411 if (xudc->genpd_dev_ss)
3412 dev_pm_domain_detach(xudc->genpd_dev_ss, true);
3413 if (xudc->genpd_dev_device)
3414 dev_pm_domain_detach(xudc->genpd_dev_device, true);
3417 static int tegra_xudc_powerdomain_init(struct tegra_xudc *xudc)
3419 struct device *dev = xudc->dev;
3422 xudc->genpd_dev_device = dev_pm_domain_attach_by_name(dev,
3424 if (IS_ERR(xudc->genpd_dev_device)) {
3425 err = PTR_ERR(xudc->genpd_dev_device);
3426 dev_err(dev, "failed to get dev pm-domain: %d\n", err);
3430 xudc->genpd_dev_ss = dev_pm_domain_attach_by_name(dev, "ss");
3431 if (IS_ERR(xudc->genpd_dev_ss)) {
3432 err = PTR_ERR(xudc->genpd_dev_ss);
3433 dev_err(dev, "failed to get superspeed pm-domain: %d\n", err);
3437 xudc->genpd_dl_device = device_link_add(dev, xudc->genpd_dev_device,
3438 DL_FLAG_PM_RUNTIME |
3440 if (!xudc->genpd_dl_device) {
3441 dev_err(dev, "adding usb device device link failed!\n");
3445 xudc->genpd_dl_ss = device_link_add(dev, xudc->genpd_dev_ss,
3446 DL_FLAG_PM_RUNTIME |
3448 if (!xudc->genpd_dl_ss) {
3449 dev_err(dev, "adding superspeed device link failed!\n");
3456 static int tegra_xudc_probe(struct platform_device *pdev)
3458 struct tegra_xudc *xudc;
3459 struct resource *res;
3460 struct usb_role_switch_desc role_sx_desc = { 0 };
3464 xudc = devm_kzalloc(&pdev->dev, sizeof(*xudc), GFP_ATOMIC);
3468 xudc->dev = &pdev->dev;
3469 platform_set_drvdata(pdev, xudc);
3471 xudc->soc = of_device_get_match_data(&pdev->dev);
3475 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "base");
3476 xudc->base = devm_ioremap_resource(&pdev->dev, res);
3477 if (IS_ERR(xudc->base))
3478 return PTR_ERR(xudc->base);
3479 xudc->phys_base = res->start;
3481 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "fpci");
3482 xudc->fpci = devm_ioremap_resource(&pdev->dev, res);
3483 if (IS_ERR(xudc->fpci))
3484 return PTR_ERR(xudc->fpci);
3486 if (xudc->soc->has_ipfs) {
3487 res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
3489 xudc->ipfs = devm_ioremap_resource(&pdev->dev, res);
3490 if (IS_ERR(xudc->ipfs))
3491 return PTR_ERR(xudc->ipfs);
3494 xudc->irq = platform_get_irq(pdev, 0);
3495 if (xudc->irq < 0) {
3496 dev_err(xudc->dev, "failed to get IRQ: %d\n",
3501 err = devm_request_irq(&pdev->dev, xudc->irq, tegra_xudc_irq, 0,
3502 dev_name(&pdev->dev), xudc);
3504 dev_err(xudc->dev, "failed to claim IRQ#%u: %d\n", xudc->irq,
3509 xudc->clks = devm_kcalloc(&pdev->dev, xudc->soc->num_clks,
3510 sizeof(*xudc->clks), GFP_KERNEL);
3514 for (i = 0; i < xudc->soc->num_clks; i++)
3515 xudc->clks[i].id = xudc->soc->clock_names[i];
3517 err = devm_clk_bulk_get(&pdev->dev, xudc->soc->num_clks,
3520 dev_err(xudc->dev, "failed to request clks %d\n", err);
3524 xudc->supplies = devm_kcalloc(&pdev->dev, xudc->soc->num_supplies,
3525 sizeof(*xudc->supplies), GFP_KERNEL);
3526 if (!xudc->supplies)
3529 for (i = 0; i < xudc->soc->num_supplies; i++)
3530 xudc->supplies[i].supply = xudc->soc->supply_names[i];
3532 err = devm_regulator_bulk_get(&pdev->dev, xudc->soc->num_supplies,
3535 dev_err(xudc->dev, "failed to request regulators %d\n", err);
3539 xudc->padctl = tegra_xusb_padctl_get(&pdev->dev);
3540 if (IS_ERR(xudc->padctl))
3541 return PTR_ERR(xudc->padctl);
3543 err = regulator_bulk_enable(xudc->soc->num_supplies, xudc->supplies);
3545 dev_err(xudc->dev, "failed to enable regulators %d\n", err);
3549 xudc->usb3_phy = devm_phy_optional_get(&pdev->dev, "usb3");
3550 if (IS_ERR(xudc->usb3_phy)) {
3551 err = PTR_ERR(xudc->usb3_phy);
3552 dev_err(xudc->dev, "failed to get usb3 phy: %d\n", err);
3553 goto disable_regulator;
3556 xudc->utmi_phy = devm_phy_optional_get(&pdev->dev, "usb2");
3557 if (IS_ERR(xudc->utmi_phy)) {
3558 err = PTR_ERR(xudc->utmi_phy);
3559 dev_err(xudc->dev, "failed to get usb2 phy: %d\n", err);
3560 goto disable_regulator;
3563 err = tegra_xudc_powerdomain_init(xudc);
3565 goto put_powerdomains;
3567 err = tegra_xudc_phy_init(xudc);
3569 goto put_powerdomains;
3571 err = tegra_xudc_alloc_event_ring(xudc);
3575 err = tegra_xudc_alloc_eps(xudc);
3577 goto free_event_ring;
3579 spin_lock_init(&xudc->lock);
3581 init_completion(&xudc->disconnect_complete);
3583 INIT_WORK(&xudc->usb_role_sw_work, tegra_xudc_usb_role_sw_work);
3585 INIT_DELAYED_WORK(&xudc->plc_reset_work, tegra_xudc_plc_reset_work);
3587 INIT_DELAYED_WORK(&xudc->port_reset_war_work,
3588 tegra_xudc_port_reset_war_work);
3590 if (of_property_read_bool(xudc->dev->of_node, "usb-role-switch")) {
3591 role_sx_desc.set = tegra_xudc_usb_role_sw_set;
3592 role_sx_desc.fwnode = dev_fwnode(xudc->dev);
3594 xudc->usb_role_sw = usb_role_switch_register(xudc->dev,
3596 if (IS_ERR(xudc->usb_role_sw)) {
3597 err = PTR_ERR(xudc->usb_role_sw);
3598 dev_err(xudc->dev, "Failed to register USB role SW: %d",
3603 /* Set the mode as device mode and this keeps phy always ON */
3604 dev_info(xudc->dev, "Set usb role to device mode always");
3605 schedule_work(&xudc->usb_role_sw_work);
3608 pm_runtime_enable(&pdev->dev);
3610 xudc->gadget.ops = &tegra_xudc_gadget_ops;
3611 xudc->gadget.ep0 = &xudc->ep[0].usb_ep;
3612 xudc->gadget.name = "tegra-xudc";
3613 xudc->gadget.max_speed = USB_SPEED_SUPER;
3615 err = usb_add_gadget_udc(&pdev->dev, &xudc->gadget);
3617 dev_err(&pdev->dev, "failed to add USB gadget: %d\n", err);
3624 tegra_xudc_free_eps(xudc);
3626 tegra_xudc_free_event_ring(xudc);
3628 tegra_xudc_phy_exit(xudc);
3630 tegra_xudc_powerdomain_remove(xudc);
3632 regulator_bulk_disable(xudc->soc->num_supplies, xudc->supplies);
3634 tegra_xusb_padctl_put(xudc->padctl);
3639 static int tegra_xudc_remove(struct platform_device *pdev)
3641 struct tegra_xudc *xudc = platform_get_drvdata(pdev);
3643 pm_runtime_get_sync(xudc->dev);
3645 cancel_delayed_work(&xudc->plc_reset_work);
3647 if (xudc->usb_role_sw) {
3648 usb_role_switch_unregister(xudc->usb_role_sw);
3649 cancel_work_sync(&xudc->usb_role_sw_work);
3652 usb_del_gadget_udc(&xudc->gadget);
3654 tegra_xudc_free_eps(xudc);
3655 tegra_xudc_free_event_ring(xudc);
3657 tegra_xudc_powerdomain_remove(xudc);
3659 regulator_bulk_disable(xudc->soc->num_supplies, xudc->supplies);
3661 phy_power_off(xudc->utmi_phy);
3662 phy_power_off(xudc->usb3_phy);
3664 tegra_xudc_phy_exit(xudc);
3666 pm_runtime_disable(xudc->dev);
3667 pm_runtime_put(xudc->dev);
3669 tegra_xusb_padctl_put(xudc->padctl);
3674 static int __maybe_unused tegra_xudc_powergate(struct tegra_xudc *xudc)
3676 unsigned long flags;
3678 dev_dbg(xudc->dev, "entering ELPG\n");
3680 spin_lock_irqsave(&xudc->lock, flags);
3682 xudc->powergated = true;
3683 xudc->saved_regs.ctrl = xudc_readl(xudc, CTRL);
3684 xudc->saved_regs.portpm = xudc_readl(xudc, PORTPM);
3685 xudc_writel(xudc, 0, CTRL);
3687 spin_unlock_irqrestore(&xudc->lock, flags);
3689 clk_bulk_disable_unprepare(xudc->soc->num_clks, xudc->clks);
3691 regulator_bulk_disable(xudc->soc->num_supplies, xudc->supplies);
3693 dev_dbg(xudc->dev, "entering ELPG done\n");
3697 static int __maybe_unused tegra_xudc_unpowergate(struct tegra_xudc *xudc)
3699 unsigned long flags;
3702 dev_dbg(xudc->dev, "exiting ELPG\n");
3704 err = regulator_bulk_enable(xudc->soc->num_supplies,
3709 err = clk_bulk_prepare_enable(xudc->soc->num_clks, xudc->clks);
3713 tegra_xudc_fpci_ipfs_init(xudc);
3715 tegra_xudc_device_params_init(xudc);
3717 tegra_xudc_init_event_ring(xudc);
3719 tegra_xudc_init_eps(xudc);
3721 xudc_writel(xudc, xudc->saved_regs.portpm, PORTPM);
3722 xudc_writel(xudc, xudc->saved_regs.ctrl, CTRL);
3724 spin_lock_irqsave(&xudc->lock, flags);
3725 xudc->powergated = false;
3726 spin_unlock_irqrestore(&xudc->lock, flags);
3728 dev_dbg(xudc->dev, "exiting ELPG done\n");
3732 static int __maybe_unused tegra_xudc_suspend(struct device *dev)
3734 struct tegra_xudc *xudc = dev_get_drvdata(dev);
3735 unsigned long flags;
3737 spin_lock_irqsave(&xudc->lock, flags);
3738 xudc->suspended = true;
3739 spin_unlock_irqrestore(&xudc->lock, flags);
3741 flush_work(&xudc->usb_role_sw_work);
3743 /* Forcibly disconnect before powergating. */
3744 tegra_xudc_device_mode_off(xudc);
3746 if (!pm_runtime_status_suspended(dev))
3747 tegra_xudc_powergate(xudc);
3749 pm_runtime_disable(dev);
3754 static int __maybe_unused tegra_xudc_resume(struct device *dev)
3756 struct tegra_xudc *xudc = dev_get_drvdata(dev);
3757 unsigned long flags;
3760 err = tegra_xudc_unpowergate(xudc);
3764 spin_lock_irqsave(&xudc->lock, flags);
3765 xudc->suspended = false;
3766 spin_unlock_irqrestore(&xudc->lock, flags);
3768 schedule_work(&xudc->usb_role_sw_work);
3770 pm_runtime_enable(dev);
3775 static int __maybe_unused tegra_xudc_runtime_suspend(struct device *dev)
3777 struct tegra_xudc *xudc = dev_get_drvdata(dev);
3779 return tegra_xudc_powergate(xudc);
3782 static int __maybe_unused tegra_xudc_runtime_resume(struct device *dev)
3784 struct tegra_xudc *xudc = dev_get_drvdata(dev);
3786 return tegra_xudc_unpowergate(xudc);
3789 static const struct dev_pm_ops tegra_xudc_pm_ops = {
3790 SET_SYSTEM_SLEEP_PM_OPS(tegra_xudc_suspend, tegra_xudc_resume)
3791 SET_RUNTIME_PM_OPS(tegra_xudc_runtime_suspend,
3792 tegra_xudc_runtime_resume, NULL)
3795 static struct platform_driver tegra_xudc_driver = {
3796 .probe = tegra_xudc_probe,
3797 .remove = tegra_xudc_remove,
3799 .name = "tegra-xudc",
3800 .pm = &tegra_xudc_pm_ops,
3801 .of_match_table = tegra_xudc_of_match,
3804 module_platform_driver(tegra_xudc_driver);
3806 MODULE_DESCRIPTION("NVIDIA Tegra XUSB Device Controller");
3807 MODULE_AUTHOR("Andrew Bresticker <abrestic@chromium.org>");
3808 MODULE_AUTHOR("Hui Fu <hfu@nvidia.com>");
3809 MODULE_AUTHOR("Nagarjuna Kristam <nkristam@nvidia.com>");
3810 MODULE_LICENSE("GPL v2");