1 // SPDX-License-Identifier: GPL-2.0
3 * Cadence CDNSP DRD Driver.
5 * Copyright (C) 2020 Cadence.
7 * Author: Pawel Laszczak <pawell@cadence.com>
11 #include <linux/moduleparam.h>
12 #include <linux/dma-mapping.h>
13 #include <linux/module.h>
14 #include <linux/iopoll.h>
15 #include <linux/delay.h>
16 #include <linux/log2.h>
17 #include <linux/slab.h>
18 #include <linux/pci.h>
19 #include <linux/irq.h>
20 #include <linux/dmi.h>
23 #include "gadget-export.h"
25 #include "cdnsp-gadget.h"
26 #include "cdnsp-trace.h"
28 unsigned int cdnsp_port_speed(unsigned int port_status)
30 /*Detect gadget speed based on PORTSC register*/
31 if (DEV_SUPERSPEEDPLUS(port_status))
32 return USB_SPEED_SUPER_PLUS;
33 else if (DEV_SUPERSPEED(port_status))
34 return USB_SPEED_SUPER;
35 else if (DEV_HIGHSPEED(port_status))
36 return USB_SPEED_HIGH;
37 else if (DEV_FULLSPEED(port_status))
38 return USB_SPEED_FULL;
40 /* If device is detached then speed will be USB_SPEED_UNKNOWN.*/
41 return USB_SPEED_UNKNOWN;
45 * Given a port state, this function returns a value that would result in the
46 * port being in the same state, if the value was written to the port status
48 * Save Read Only (RO) bits and save read/write bits where
49 * writing a 0 clears the bit and writing a 1 sets the bit (RWS).
50 * For all other types (RW1S, RW1CS, RW, and RZ), writing a '0' has no effect.
52 u32 cdnsp_port_state_to_neutral(u32 state)
54 /* Save read-only status and port state. */
55 return (state & CDNSP_PORT_RO) | (state & CDNSP_PORT_RWS);
59 * cdnsp_find_next_ext_cap - Find the offset of the extended capabilities
60 * with capability ID id.
61 * @base: PCI MMIO registers base address.
62 * @start: Address at which to start looking, (0 or HCC_PARAMS to start at
64 * @id: Extended capability ID to search for.
66 * Returns the offset of the next matching extended capability structure.
67 * Some capabilities can occur several times,
68 * e.g., the EXT_CAPS_PROTOCOL, and this provides a way to find them all.
70 int cdnsp_find_next_ext_cap(void __iomem *base, u32 start, int id)
76 if (!start || start == HCC_PARAMS_OFFSET) {
77 val = readl(base + HCC_PARAMS_OFFSET);
81 offset = HCC_EXT_CAPS(val) << 2;
87 val = readl(base + offset);
91 if (EXT_CAPS_ID(val) == id && offset != start)
94 next = EXT_CAPS_NEXT(val);
101 void cdnsp_set_link_state(struct cdnsp_device *pdev,
102 __le32 __iomem *port_regs,
108 temp = readl(port_regs);
109 temp = cdnsp_port_state_to_neutral(temp);
110 temp |= PORT_WKCONN_E | PORT_WKDISC_E;
111 writel(temp, port_regs);
113 temp &= ~PORT_PLS_MASK;
114 temp |= PORT_LINK_STROBE | link_state;
116 if (pdev->active_port)
117 port_num = pdev->active_port->port_num;
119 trace_cdnsp_handle_port_status(port_num, readl(port_regs));
120 writel(temp, port_regs);
121 trace_cdnsp_link_state_changed(port_num, readl(port_regs));
124 static void cdnsp_disable_port(struct cdnsp_device *pdev,
125 __le32 __iomem *port_regs)
127 u32 temp = cdnsp_port_state_to_neutral(readl(port_regs));
129 writel(temp | PORT_PED, port_regs);
132 static void cdnsp_clear_port_change_bit(struct cdnsp_device *pdev,
133 __le32 __iomem *port_regs)
135 u32 portsc = readl(port_regs);
137 writel(cdnsp_port_state_to_neutral(portsc) |
138 (portsc & PORT_CHANGE_BITS), port_regs);
141 static void cdnsp_set_chicken_bits_2(struct cdnsp_device *pdev, u32 bit)
147 base = &pdev->cap_regs->hc_capbase;
148 offset = cdnsp_find_next_ext_cap(base, offset, D_XEC_PRE_REGS_CAP);
149 reg = base + offset + REG_CHICKEN_BITS_2_OFFSET;
151 bit = readl(reg) | bit;
155 static void cdnsp_clear_chicken_bits_2(struct cdnsp_device *pdev, u32 bit)
161 base = &pdev->cap_regs->hc_capbase;
162 offset = cdnsp_find_next_ext_cap(base, offset, D_XEC_PRE_REGS_CAP);
163 reg = base + offset + REG_CHICKEN_BITS_2_OFFSET;
165 bit = readl(reg) & ~bit;
170 * Disable interrupts and begin the controller halting process.
172 static void cdnsp_quiesce(struct cdnsp_device *pdev)
178 mask = ~(u32)(CDNSP_IRQS);
180 halted = readl(&pdev->op_regs->status) & STS_HALT;
182 mask &= ~(CMD_R_S | CMD_DEVEN);
184 cmd = readl(&pdev->op_regs->command);
186 writel(cmd, &pdev->op_regs->command);
190 * Force controller into halt state.
192 * Disable any IRQs and clear the run/stop bit.
193 * Controller will complete any current and actively pipelined transactions, and
194 * should halt within 16 ms of the run/stop bit being cleared.
195 * Read controller Halted bit in the status register to see when the
196 * controller is finished.
198 int cdnsp_halt(struct cdnsp_device *pdev)
205 ret = readl_poll_timeout_atomic(&pdev->op_regs->status, val,
207 CDNSP_MAX_HALT_USEC);
209 dev_err(pdev->dev, "ERROR: Device halt failed\n");
213 pdev->cdnsp_state |= CDNSP_STATE_HALTED;
219 * device controller died, register read returns 0xffffffff, or command never
222 void cdnsp_died(struct cdnsp_device *pdev)
224 dev_err(pdev->dev, "ERROR: CDNSP controller not responding\n");
225 pdev->cdnsp_state |= CDNSP_STATE_DYING;
230 * Set the run bit and wait for the device to be running.
232 static int cdnsp_start(struct cdnsp_device *pdev)
237 temp = readl(&pdev->op_regs->command);
238 temp |= (CMD_R_S | CMD_DEVEN);
239 writel(temp, &pdev->op_regs->command);
241 pdev->cdnsp_state = 0;
244 * Wait for the STS_HALT Status bit to be 0 to indicate the device is
247 ret = readl_poll_timeout_atomic(&pdev->op_regs->status, temp,
248 !(temp & STS_HALT), 1,
249 CDNSP_MAX_HALT_USEC);
251 pdev->cdnsp_state = CDNSP_STATE_DYING;
252 dev_err(pdev->dev, "ERROR: Controller run failed\n");
259 * Reset a halted controller.
261 * This resets pipelines, timers, counters, state machines, etc.
262 * Transactions will be terminated immediately, and operational registers
263 * will be set to their defaults.
265 int cdnsp_reset(struct cdnsp_device *pdev)
271 temp = readl(&pdev->op_regs->status);
273 if (temp == ~(u32)0) {
274 dev_err(pdev->dev, "Device not accessible, reset failed.\n");
278 if ((temp & STS_HALT) == 0) {
279 dev_err(pdev->dev, "Controller not halted, aborting reset.\n");
283 command = readl(&pdev->op_regs->command);
284 command |= CMD_RESET;
285 writel(command, &pdev->op_regs->command);
287 ret = readl_poll_timeout_atomic(&pdev->op_regs->command, temp,
288 !(temp & CMD_RESET), 1,
291 dev_err(pdev->dev, "ERROR: Controller reset failed\n");
296 * CDNSP cannot write any doorbells or operational registers other
297 * than status until the "Controller Not Ready" flag is cleared.
299 ret = readl_poll_timeout_atomic(&pdev->op_regs->status, temp,
300 !(temp & STS_CNR), 1,
304 dev_err(pdev->dev, "ERROR: Controller not ready to work\n");
308 dev_dbg(pdev->dev, "Controller ready to work");
314 * cdnsp_get_endpoint_index - Find the index for an endpoint given its
315 * descriptor.Use the return value to right shift 1 for the bitmask.
317 * Index = (epnum * 2) + direction - 1,
318 * where direction = 0 for OUT, 1 for IN.
319 * For control endpoints, the IN index is used (OUT index is unused), so
320 * index = (epnum * 2) + direction - 1 = (epnum * 2) + 1 - 1 = (epnum * 2)
323 cdnsp_get_endpoint_index(const struct usb_endpoint_descriptor *desc)
325 unsigned int index = (unsigned int)usb_endpoint_num(desc);
327 if (usb_endpoint_xfer_control(desc))
330 return (index * 2) + (usb_endpoint_dir_in(desc) ? 1 : 0) - 1;
334 * Find the flag for this endpoint (for use in the control context). Use the
335 * endpoint index to create a bitmask. The slot context is bit 0, endpoint 0 is
339 cdnsp_get_endpoint_flag(const struct usb_endpoint_descriptor *desc)
341 return 1 << (cdnsp_get_endpoint_index(desc) + 1);
344 int cdnsp_ep_enqueue(struct cdnsp_ep *pep, struct cdnsp_request *preq)
346 struct cdnsp_device *pdev = pep->pdev;
347 struct usb_request *request;
350 if (preq->epnum == 0 && !list_empty(&pep->pending_list)) {
351 trace_cdnsp_request_enqueue_busy(preq);
355 request = &preq->request;
357 request->status = -EINPROGRESS;
358 preq->direction = pep->direction;
359 preq->epnum = pep->number;
362 ret = usb_gadget_map_request_by_dev(pdev->dev, request, pep->direction);
364 trace_cdnsp_request_enqueue_error(preq);
368 list_add_tail(&preq->list, &pep->pending_list);
370 trace_cdnsp_request_enqueue(preq);
372 switch (usb_endpoint_type(pep->endpoint.desc)) {
373 case USB_ENDPOINT_XFER_CONTROL:
374 ret = cdnsp_queue_ctrl_tx(pdev, preq);
376 case USB_ENDPOINT_XFER_BULK:
377 case USB_ENDPOINT_XFER_INT:
378 ret = cdnsp_queue_bulk_tx(pdev, preq);
380 case USB_ENDPOINT_XFER_ISOC:
381 ret = cdnsp_queue_isoc_tx(pdev, preq);
390 usb_gadget_unmap_request_by_dev(pdev->dev, &preq->request,
392 list_del(&preq->list);
393 trace_cdnsp_request_enqueue_error(preq);
399 * Remove the request's TD from the endpoint ring. This may cause the
400 * controller to stop USB transfers, potentially stopping in the middle of a
401 * TRB buffer. The controller should pick up where it left off in the TD,
402 * unless a Set Transfer Ring Dequeue Pointer is issued.
404 * The TRBs that make up the buffers for the canceled request will be "removed"
405 * from the ring. Since the ring is a contiguous structure, they can't be
406 * physically removed. Instead, there are two options:
408 * 1) If the controller is in the middle of processing the request to be
409 * canceled, we simply move the ring's dequeue pointer past those TRBs
410 * using the Set Transfer Ring Dequeue Pointer command. This will be
411 * the common case, when drivers timeout on the last submitted request
412 * and attempt to cancel.
414 * 2) If the controller is in the middle of a different TD, we turn the TRBs
415 * into a series of 1-TRB transfer no-op TDs. No-ops shouldn't be chained.
416 * The controller will need to invalidate the any TRBs it has cached after
417 * the stop endpoint command.
419 * 3) The TD may have completed by the time the Stop Endpoint Command
420 * completes, so software needs to handle that case too.
423 int cdnsp_ep_dequeue(struct cdnsp_ep *pep, struct cdnsp_request *preq)
425 struct cdnsp_device *pdev = pep->pdev;
429 trace_cdnsp_request_dequeue(preq);
431 if (GET_EP_CTX_STATE(pep->out_ctx) == EP_STATE_RUNNING)
432 ret_stop = cdnsp_cmd_stop_ep(pdev, pep);
434 ret_rem = cdnsp_remove_request(pdev, preq, pep);
436 return ret_rem ? ret_rem : ret_stop;
439 static void cdnsp_zero_in_ctx(struct cdnsp_device *pdev)
441 struct cdnsp_input_control_ctx *ctrl_ctx;
442 struct cdnsp_slot_ctx *slot_ctx;
443 struct cdnsp_ep_ctx *ep_ctx;
446 ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
449 * When a device's add flag and drop flag are zero, any subsequent
450 * configure endpoint command will leave that endpoint's state
451 * untouched. Make sure we don't leave any old state in the input
454 ctrl_ctx->drop_flags = 0;
455 ctrl_ctx->add_flags = 0;
456 slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx);
457 slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);
459 /* Endpoint 0 is always valid */
460 slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(1));
461 for (i = 1; i < CDNSP_ENDPOINTS_NUM; ++i) {
462 ep_ctx = cdnsp_get_ep_ctx(&pdev->in_ctx, i);
464 ep_ctx->ep_info2 = 0;
470 /* Issue a configure endpoint command and wait for it to finish. */
471 static int cdnsp_configure_endpoint(struct cdnsp_device *pdev)
475 cdnsp_queue_configure_endpoint(pdev, pdev->cmd.in_ctx->dma);
476 cdnsp_ring_cmd_db(pdev);
477 ret = cdnsp_wait_for_cmd_compl(pdev);
480 "ERR: unexpected command completion code 0x%x.\n", ret);
487 static void cdnsp_invalidate_ep_events(struct cdnsp_device *pdev,
488 struct cdnsp_ep *pep)
490 struct cdnsp_segment *segment;
491 union cdnsp_trb *event;
495 event = pdev->event_ring->dequeue;
496 segment = pdev->event_ring->deq_seg;
497 cycle_state = pdev->event_ring->cycle_state;
500 data = le32_to_cpu(event->trans_event.flags);
502 /* Check the owner of the TRB. */
503 if ((data & TRB_CYCLE) != cycle_state)
506 if (TRB_FIELD_TO_TYPE(data) == TRB_TRANSFER &&
507 TRB_TO_EP_ID(data) == (pep->idx + 1)) {
508 data |= TRB_EVENT_INVALIDATE;
509 event->trans_event.flags = cpu_to_le32(data);
512 if (cdnsp_last_trb_on_seg(segment, event)) {
514 segment = pdev->event_ring->deq_seg->next;
515 event = segment->trbs;
522 int cdnsp_wait_for_cmd_compl(struct cdnsp_device *pdev)
524 struct cdnsp_segment *event_deq_seg;
525 union cdnsp_trb *cmd_trb;
526 dma_addr_t cmd_deq_dma;
527 union cdnsp_trb *event;
533 cmd_trb = pdev->cmd.command_trb;
534 pdev->cmd.status = 0;
536 trace_cdnsp_cmd_wait_for_compl(pdev->cmd_ring, &cmd_trb->generic);
538 ret = readl_poll_timeout_atomic(&pdev->op_regs->cmd_ring, val,
539 !CMD_RING_BUSY(val), 1,
542 dev_err(pdev->dev, "ERR: Timeout while waiting for command\n");
543 trace_cdnsp_cmd_timeout(pdev->cmd_ring, &cmd_trb->generic);
544 pdev->cdnsp_state = CDNSP_STATE_DYING;
548 event = pdev->event_ring->dequeue;
549 event_deq_seg = pdev->event_ring->deq_seg;
550 cycle_state = pdev->event_ring->cycle_state;
552 cmd_deq_dma = cdnsp_trb_virt_to_dma(pdev->cmd_ring->deq_seg, cmd_trb);
557 flags = le32_to_cpu(event->event_cmd.flags);
559 /* Check the owner of the TRB. */
560 if ((flags & TRB_CYCLE) != cycle_state)
563 cmd_dma = le64_to_cpu(event->event_cmd.cmd_trb);
566 * Check whether the completion event is for last queued
569 if (TRB_FIELD_TO_TYPE(flags) != TRB_COMPLETION ||
570 cmd_dma != (u64)cmd_deq_dma) {
571 if (!cdnsp_last_trb_on_seg(event_deq_seg, event)) {
576 if (cdnsp_last_trb_on_ring(pdev->event_ring,
577 event_deq_seg, event))
580 event_deq_seg = event_deq_seg->next;
581 event = event_deq_seg->trbs;
585 trace_cdnsp_handle_command(pdev->cmd_ring, &cmd_trb->generic);
587 pdev->cmd.status = GET_COMP_CODE(le32_to_cpu(event->event_cmd.status));
588 if (pdev->cmd.status == COMP_SUCCESS)
591 return -pdev->cmd.status;
595 int cdnsp_halt_endpoint(struct cdnsp_device *pdev,
596 struct cdnsp_ep *pep,
601 trace_cdnsp_ep_halt(value ? "Set" : "Clear");
603 ret = cdnsp_cmd_stop_ep(pdev, pep);
608 if (GET_EP_CTX_STATE(pep->out_ctx) == EP_STATE_STOPPED) {
609 cdnsp_queue_halt_endpoint(pdev, pep->idx);
610 cdnsp_ring_cmd_db(pdev);
611 ret = cdnsp_wait_for_cmd_compl(pdev);
614 pep->ep_state |= EP_HALTED;
616 cdnsp_queue_reset_ep(pdev, pep->idx);
617 cdnsp_ring_cmd_db(pdev);
618 ret = cdnsp_wait_for_cmd_compl(pdev);
619 trace_cdnsp_handle_cmd_reset_ep(pep->out_ctx);
624 pep->ep_state &= ~EP_HALTED;
626 if (pep->idx != 0 && !(pep->ep_state & EP_WEDGE))
627 cdnsp_ring_doorbell_for_active_rings(pdev, pep);
629 pep->ep_state &= ~EP_WEDGE;
635 static int cdnsp_update_eps_configuration(struct cdnsp_device *pdev,
636 struct cdnsp_ep *pep)
638 struct cdnsp_input_control_ctx *ctrl_ctx;
639 struct cdnsp_slot_ctx *slot_ctx;
644 ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
646 /* Don't issue the command if there's no endpoints to update. */
647 if (ctrl_ctx->add_flags == 0 && ctrl_ctx->drop_flags == 0)
650 ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
651 ctrl_ctx->add_flags &= cpu_to_le32(~EP0_FLAG);
652 ctrl_ctx->drop_flags &= cpu_to_le32(~(SLOT_FLAG | EP0_FLAG));
654 /* Fix up Context Entries field. Minimum value is EP0 == BIT(1). */
655 slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx);
656 for (i = CDNSP_ENDPOINTS_NUM; i >= 1; i--) {
657 __le32 le32 = cpu_to_le32(BIT(i));
659 if ((pdev->eps[i - 1].ring && !(ctrl_ctx->drop_flags & le32)) ||
660 (ctrl_ctx->add_flags & le32) || i == 1) {
661 slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);
662 slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(i));
667 ep_sts = GET_EP_CTX_STATE(pep->out_ctx);
669 if ((ctrl_ctx->add_flags != cpu_to_le32(SLOT_FLAG) &&
670 ep_sts == EP_STATE_DISABLED) ||
671 (ep_sts != EP_STATE_DISABLED && ctrl_ctx->drop_flags))
672 ret = cdnsp_configure_endpoint(pdev);
674 trace_cdnsp_configure_endpoint(cdnsp_get_slot_ctx(&pdev->out_ctx));
675 trace_cdnsp_handle_cmd_config_ep(pep->out_ctx);
677 cdnsp_zero_in_ctx(pdev);
683 * This submits a Reset Device Command, which will set the device state to 0,
684 * set the device address to 0, and disable all the endpoints except the default
685 * control endpoint. The USB core should come back and call
686 * cdnsp_setup_device(), and then re-set up the configuration.
688 int cdnsp_reset_device(struct cdnsp_device *pdev)
690 struct cdnsp_slot_ctx *slot_ctx;
694 slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx);
695 slot_ctx->dev_info = 0;
696 pdev->device_address = 0;
698 /* If device is not setup, there is no point in resetting it. */
699 slot_ctx = cdnsp_get_slot_ctx(&pdev->out_ctx);
700 slot_state = GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state));
701 trace_cdnsp_reset_device(slot_ctx);
703 if (slot_state <= SLOT_STATE_DEFAULT &&
704 pdev->eps[0].ep_state & EP_HALTED) {
705 cdnsp_halt_endpoint(pdev, &pdev->eps[0], 0);
709 * During Reset Device command controller shall transition the
710 * endpoint ep0 to the Running State.
712 pdev->eps[0].ep_state &= ~(EP_STOPPED | EP_HALTED);
713 pdev->eps[0].ep_state |= EP_ENABLED;
715 if (slot_state <= SLOT_STATE_DEFAULT)
718 cdnsp_queue_reset_device(pdev);
719 cdnsp_ring_cmd_db(pdev);
720 ret = cdnsp_wait_for_cmd_compl(pdev);
723 * After Reset Device command all not default endpoints
724 * are in Disabled state.
726 for (i = 1; i < CDNSP_ENDPOINTS_NUM; ++i)
727 pdev->eps[i].ep_state |= EP_STOPPED | EP_UNCONFIGURED;
729 trace_cdnsp_handle_cmd_reset_dev(slot_ctx);
732 dev_err(pdev->dev, "Reset device failed with error code %d",
739 * Sets the MaxPStreams field and the Linear Stream Array field.
740 * Sets the dequeue pointer to the stream context array.
742 static void cdnsp_setup_streams_ep_input_ctx(struct cdnsp_device *pdev,
743 struct cdnsp_ep_ctx *ep_ctx,
744 struct cdnsp_stream_info *stream_info)
746 u32 max_primary_streams;
748 /* MaxPStreams is the number of stream context array entries, not the
749 * number we're actually using. Must be in 2^(MaxPstreams + 1) format.
750 * fls(0) = 0, fls(0x1) = 1, fls(0x10) = 2, fls(0x100) = 3, etc.
752 max_primary_streams = fls(stream_info->num_stream_ctxs) - 2;
753 ep_ctx->ep_info &= cpu_to_le32(~EP_MAXPSTREAMS_MASK);
754 ep_ctx->ep_info |= cpu_to_le32(EP_MAXPSTREAMS(max_primary_streams)
756 ep_ctx->deq = cpu_to_le64(stream_info->ctx_array_dma);
760 * The drivers use this function to prepare a bulk endpoints to use streams.
762 * Don't allow the call to succeed if endpoint only supports one stream
763 * (which means it doesn't support streams at all).
765 int cdnsp_alloc_streams(struct cdnsp_device *pdev, struct cdnsp_ep *pep)
767 unsigned int num_streams = usb_ss_max_streams(pep->endpoint.comp_desc);
768 unsigned int num_stream_ctxs;
771 if (num_streams == 0)
774 if (num_streams > STREAM_NUM_STREAMS)
778 * Add two to the number of streams requested to account for
779 * stream 0 that is reserved for controller usage and one additional
780 * for TASK SET FULL response.
784 /* The stream context array size must be a power of two */
785 num_stream_ctxs = roundup_pow_of_two(num_streams);
787 trace_cdnsp_stream_number(pep, num_stream_ctxs, num_streams);
789 ret = cdnsp_alloc_stream_info(pdev, pep, num_stream_ctxs, num_streams);
793 cdnsp_setup_streams_ep_input_ctx(pdev, pep->in_ctx, &pep->stream_info);
795 pep->ep_state |= EP_HAS_STREAMS;
796 pep->stream_info.td_count = 0;
797 pep->stream_info.first_prime_det = 0;
799 /* Subtract 1 for stream 0, which drivers can't use. */
800 return num_streams - 1;
803 int cdnsp_disable_slot(struct cdnsp_device *pdev)
807 cdnsp_queue_slot_control(pdev, TRB_DISABLE_SLOT);
808 cdnsp_ring_cmd_db(pdev);
809 ret = cdnsp_wait_for_cmd_compl(pdev);
812 pdev->active_port = NULL;
814 trace_cdnsp_handle_cmd_disable_slot(cdnsp_get_slot_ctx(&pdev->out_ctx));
816 memset(pdev->in_ctx.bytes, 0, CDNSP_CTX_SIZE);
817 memset(pdev->out_ctx.bytes, 0, CDNSP_CTX_SIZE);
822 int cdnsp_enable_slot(struct cdnsp_device *pdev)
824 struct cdnsp_slot_ctx *slot_ctx;
828 /* If device is not setup, there is no point in resetting it */
829 slot_ctx = cdnsp_get_slot_ctx(&pdev->out_ctx);
830 slot_state = GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state));
832 if (slot_state != SLOT_STATE_DISABLED)
835 cdnsp_queue_slot_control(pdev, TRB_ENABLE_SLOT);
836 cdnsp_ring_cmd_db(pdev);
837 ret = cdnsp_wait_for_cmd_compl(pdev);
844 trace_cdnsp_handle_cmd_enable_slot(cdnsp_get_slot_ctx(&pdev->out_ctx));
850 * Issue an Address Device command with BSR=0 if setup is SETUP_CONTEXT_ONLY
851 * or with BSR = 1 if set_address is SETUP_CONTEXT_ADDRESS.
853 int cdnsp_setup_device(struct cdnsp_device *pdev, enum cdnsp_setup_dev setup)
855 struct cdnsp_input_control_ctx *ctrl_ctx;
856 struct cdnsp_slot_ctx *slot_ctx;
860 if (!pdev->slot_id) {
861 trace_cdnsp_slot_id("incorrect");
865 if (!pdev->active_port->port_num)
868 slot_ctx = cdnsp_get_slot_ctx(&pdev->out_ctx);
869 dev_state = GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state));
871 if (setup == SETUP_CONTEXT_ONLY && dev_state == SLOT_STATE_DEFAULT) {
872 trace_cdnsp_slot_already_in_default(slot_ctx);
876 slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx);
877 ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
879 if (!slot_ctx->dev_info || dev_state == SLOT_STATE_DEFAULT) {
880 ret = cdnsp_setup_addressable_priv_dev(pdev);
885 cdnsp_copy_ep0_dequeue_into_input_ctx(pdev);
887 ctrl_ctx->add_flags = cpu_to_le32(SLOT_FLAG | EP0_FLAG);
888 ctrl_ctx->drop_flags = 0;
890 trace_cdnsp_setup_device_slot(slot_ctx);
892 cdnsp_queue_address_device(pdev, pdev->in_ctx.dma, setup);
893 cdnsp_ring_cmd_db(pdev);
894 ret = cdnsp_wait_for_cmd_compl(pdev);
896 trace_cdnsp_handle_cmd_addr_dev(cdnsp_get_slot_ctx(&pdev->out_ctx));
898 /* Zero the input context control for later use. */
899 ctrl_ctx->add_flags = 0;
900 ctrl_ctx->drop_flags = 0;
905 void cdnsp_set_usb2_hardware_lpm(struct cdnsp_device *pdev,
906 struct usb_request *req,
909 if (pdev->active_port != &pdev->usb2_port || !pdev->gadget.lpm_capable)
912 trace_cdnsp_lpm(enable);
915 writel(PORT_BESL(CDNSP_DEFAULT_BESL) | PORT_L1S_NYET | PORT_HLE,
916 &pdev->active_port->regs->portpmsc);
918 writel(PORT_L1S_NYET, &pdev->active_port->regs->portpmsc);
921 static int cdnsp_get_frame(struct cdnsp_device *pdev)
923 return readl(&pdev->run_regs->microframe_index) >> 3;
926 static int cdnsp_gadget_ep_enable(struct usb_ep *ep,
927 const struct usb_endpoint_descriptor *desc)
929 struct cdnsp_input_control_ctx *ctrl_ctx;
930 struct cdnsp_device *pdev;
931 struct cdnsp_ep *pep;
936 if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT ||
937 !desc->wMaxPacketSize)
940 pep = to_cdnsp_ep(ep);
942 pep->ep_state &= ~EP_UNCONFIGURED;
944 if (dev_WARN_ONCE(pdev->dev, pep->ep_state & EP_ENABLED,
945 "%s is already enabled\n", pep->name))
948 spin_lock_irqsave(&pdev->lock, flags);
950 added_ctxs = cdnsp_get_endpoint_flag(desc);
951 if (added_ctxs == SLOT_FLAG || added_ctxs == EP0_FLAG) {
952 dev_err(pdev->dev, "ERROR: Bad endpoint number\n");
957 pep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
959 if (pdev->gadget.speed == USB_SPEED_FULL) {
960 if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_INT)
961 pep->interval = desc->bInterval << 3;
962 if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_ISOC)
963 pep->interval = BIT(desc->bInterval - 1) << 3;
966 if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_ISOC) {
967 if (pep->interval > BIT(12)) {
968 dev_err(pdev->dev, "bInterval %d not supported\n",
973 cdnsp_set_chicken_bits_2(pdev, CHICKEN_XDMA_2_TP_CACHE_DIS);
976 ret = cdnsp_endpoint_init(pdev, pep, GFP_ATOMIC);
980 ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
981 ctrl_ctx->add_flags = cpu_to_le32(added_ctxs);
982 ctrl_ctx->drop_flags = 0;
984 ret = cdnsp_update_eps_configuration(pdev, pep);
986 cdnsp_free_endpoint_rings(pdev, pep);
990 pep->ep_state |= EP_ENABLED;
991 pep->ep_state &= ~EP_STOPPED;
994 trace_cdnsp_ep_enable_end(pep, 0);
995 spin_unlock_irqrestore(&pdev->lock, flags);
1000 static int cdnsp_gadget_ep_disable(struct usb_ep *ep)
1002 struct cdnsp_input_control_ctx *ctrl_ctx;
1003 struct cdnsp_request *preq;
1004 struct cdnsp_device *pdev;
1005 struct cdnsp_ep *pep;
1006 unsigned long flags;
1013 pep = to_cdnsp_ep(ep);
1016 spin_lock_irqsave(&pdev->lock, flags);
1018 if (!(pep->ep_state & EP_ENABLED)) {
1019 dev_err(pdev->dev, "%s is already disabled\n", pep->name);
1024 pep->ep_state |= EP_DIS_IN_RROGRESS;
1026 /* Endpoint was unconfigured by Reset Device command. */
1027 if (!(pep->ep_state & EP_UNCONFIGURED)) {
1028 cdnsp_cmd_stop_ep(pdev, pep);
1029 cdnsp_cmd_flush_ep(pdev, pep);
1032 /* Remove all queued USB requests. */
1033 while (!list_empty(&pep->pending_list)) {
1034 preq = next_request(&pep->pending_list);
1035 cdnsp_ep_dequeue(pep, preq);
1038 cdnsp_invalidate_ep_events(pdev, pep);
1040 pep->ep_state &= ~EP_DIS_IN_RROGRESS;
1041 drop_flag = cdnsp_get_endpoint_flag(pep->endpoint.desc);
1042 ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
1043 ctrl_ctx->drop_flags = cpu_to_le32(drop_flag);
1044 ctrl_ctx->add_flags = 0;
1046 cdnsp_endpoint_zero(pdev, pep);
1048 if (!(pep->ep_state & EP_UNCONFIGURED))
1049 ret = cdnsp_update_eps_configuration(pdev, pep);
1051 cdnsp_free_endpoint_rings(pdev, pep);
1053 pep->ep_state &= ~(EP_ENABLED | EP_UNCONFIGURED);
1054 pep->ep_state |= EP_STOPPED;
1057 trace_cdnsp_ep_disable_end(pep, 0);
1058 spin_unlock_irqrestore(&pdev->lock, flags);
1063 static struct usb_request *cdnsp_gadget_ep_alloc_request(struct usb_ep *ep,
1066 struct cdnsp_ep *pep = to_cdnsp_ep(ep);
1067 struct cdnsp_request *preq;
1069 preq = kzalloc(sizeof(*preq), gfp_flags);
1073 preq->epnum = pep->number;
1076 trace_cdnsp_alloc_request(preq);
1078 return &preq->request;
1081 static void cdnsp_gadget_ep_free_request(struct usb_ep *ep,
1082 struct usb_request *request)
1084 struct cdnsp_request *preq = to_cdnsp_request(request);
1086 trace_cdnsp_free_request(preq);
1090 static int cdnsp_gadget_ep_queue(struct usb_ep *ep,
1091 struct usb_request *request,
1094 struct cdnsp_request *preq;
1095 struct cdnsp_device *pdev;
1096 struct cdnsp_ep *pep;
1097 unsigned long flags;
1100 if (!request || !ep)
1103 pep = to_cdnsp_ep(ep);
1106 if (!(pep->ep_state & EP_ENABLED)) {
1107 dev_err(pdev->dev, "%s: can't queue to disabled endpoint\n",
1112 preq = to_cdnsp_request(request);
1113 spin_lock_irqsave(&pdev->lock, flags);
1114 ret = cdnsp_ep_enqueue(pep, preq);
1115 spin_unlock_irqrestore(&pdev->lock, flags);
1120 static int cdnsp_gadget_ep_dequeue(struct usb_ep *ep,
1121 struct usb_request *request)
1123 struct cdnsp_ep *pep = to_cdnsp_ep(ep);
1124 struct cdnsp_device *pdev = pep->pdev;
1125 unsigned long flags;
1128 if (request->status != -EINPROGRESS)
1131 if (!pep->endpoint.desc) {
1133 "%s: can't dequeue to disabled endpoint\n",
1138 /* Requests has been dequeued during disabling endpoint. */
1139 if (!(pep->ep_state & EP_ENABLED))
1142 spin_lock_irqsave(&pdev->lock, flags);
1143 ret = cdnsp_ep_dequeue(pep, to_cdnsp_request(request));
1144 spin_unlock_irqrestore(&pdev->lock, flags);
1149 static int cdnsp_gadget_ep_set_halt(struct usb_ep *ep, int value)
1151 struct cdnsp_ep *pep = to_cdnsp_ep(ep);
1152 struct cdnsp_device *pdev = pep->pdev;
1153 struct cdnsp_request *preq;
1154 unsigned long flags;
1157 spin_lock_irqsave(&pdev->lock, flags);
1159 preq = next_request(&pep->pending_list);
1162 trace_cdnsp_ep_busy_try_halt_again(pep, 0);
1168 ret = cdnsp_halt_endpoint(pdev, pep, value);
1171 spin_unlock_irqrestore(&pdev->lock, flags);
1175 static int cdnsp_gadget_ep_set_wedge(struct usb_ep *ep)
1177 struct cdnsp_ep *pep = to_cdnsp_ep(ep);
1178 struct cdnsp_device *pdev = pep->pdev;
1179 unsigned long flags;
1182 spin_lock_irqsave(&pdev->lock, flags);
1183 pep->ep_state |= EP_WEDGE;
1184 ret = cdnsp_halt_endpoint(pdev, pep, 1);
1185 spin_unlock_irqrestore(&pdev->lock, flags);
1190 static const struct usb_ep_ops cdnsp_gadget_ep0_ops = {
1191 .enable = cdnsp_gadget_ep_enable,
1192 .disable = cdnsp_gadget_ep_disable,
1193 .alloc_request = cdnsp_gadget_ep_alloc_request,
1194 .free_request = cdnsp_gadget_ep_free_request,
1195 .queue = cdnsp_gadget_ep_queue,
1196 .dequeue = cdnsp_gadget_ep_dequeue,
1197 .set_halt = cdnsp_gadget_ep_set_halt,
1198 .set_wedge = cdnsp_gadget_ep_set_wedge,
1201 static const struct usb_ep_ops cdnsp_gadget_ep_ops = {
1202 .enable = cdnsp_gadget_ep_enable,
1203 .disable = cdnsp_gadget_ep_disable,
1204 .alloc_request = cdnsp_gadget_ep_alloc_request,
1205 .free_request = cdnsp_gadget_ep_free_request,
1206 .queue = cdnsp_gadget_ep_queue,
1207 .dequeue = cdnsp_gadget_ep_dequeue,
1208 .set_halt = cdnsp_gadget_ep_set_halt,
1209 .set_wedge = cdnsp_gadget_ep_set_wedge,
1212 void cdnsp_gadget_giveback(struct cdnsp_ep *pep,
1213 struct cdnsp_request *preq,
1216 struct cdnsp_device *pdev = pep->pdev;
1218 list_del(&preq->list);
1220 if (preq->request.status == -EINPROGRESS)
1221 preq->request.status = status;
1223 usb_gadget_unmap_request_by_dev(pdev->dev, &preq->request,
1226 trace_cdnsp_request_giveback(preq);
1228 if (preq != &pdev->ep0_preq) {
1229 spin_unlock(&pdev->lock);
1230 usb_gadget_giveback_request(&pep->endpoint, &preq->request);
1231 spin_lock(&pdev->lock);
1235 static struct usb_endpoint_descriptor cdnsp_gadget_ep0_desc = {
1236 .bLength = USB_DT_ENDPOINT_SIZE,
1237 .bDescriptorType = USB_DT_ENDPOINT,
1238 .bmAttributes = USB_ENDPOINT_XFER_CONTROL,
1241 static int cdnsp_run(struct cdnsp_device *pdev,
1242 enum usb_device_speed speed)
1248 temp = readl(&pdev->ir_set->irq_control);
1249 temp &= ~IMOD_INTERVAL_MASK;
1250 temp |= ((IMOD_DEFAULT_INTERVAL / 250) & IMOD_INTERVAL_MASK);
1251 writel(temp, &pdev->ir_set->irq_control);
1253 temp = readl(&pdev->port3x_regs->mode_addr);
1256 case USB_SPEED_SUPER_PLUS:
1257 temp |= CFG_3XPORT_SSP_SUPPORT;
1259 case USB_SPEED_SUPER:
1260 temp &= ~CFG_3XPORT_SSP_SUPPORT;
1262 case USB_SPEED_HIGH:
1264 case USB_SPEED_FULL:
1265 fs_speed = PORT_REG6_FORCE_FS;
1268 dev_err(pdev->dev, "invalid maximum_speed parameter %d\n",
1271 case USB_SPEED_UNKNOWN:
1272 /* Default to superspeed. */
1273 speed = USB_SPEED_SUPER;
1277 if (speed >= USB_SPEED_SUPER) {
1278 writel(temp, &pdev->port3x_regs->mode_addr);
1279 cdnsp_set_link_state(pdev, &pdev->usb3_port.regs->portsc,
1282 cdnsp_disable_port(pdev, &pdev->usb3_port.regs->portsc);
1285 cdnsp_set_link_state(pdev, &pdev->usb2_port.regs->portsc,
1288 cdnsp_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1290 writel(PORT_REG6_L1_L0_HW_EN | fs_speed, &pdev->port20_regs->port_reg6);
1292 ret = cdnsp_start(pdev);
1298 temp = readl(&pdev->op_regs->command);
1300 writel(temp, &pdev->op_regs->command);
1302 temp = readl(&pdev->ir_set->irq_pending);
1303 writel(IMAN_IE_SET(temp), &pdev->ir_set->irq_pending);
1305 trace_cdnsp_init("Controller ready to work");
1312 static int cdnsp_gadget_udc_start(struct usb_gadget *g,
1313 struct usb_gadget_driver *driver)
1315 enum usb_device_speed max_speed = driver->max_speed;
1316 struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1317 unsigned long flags;
1320 spin_lock_irqsave(&pdev->lock, flags);
1321 pdev->gadget_driver = driver;
1323 /* limit speed if necessary */
1324 max_speed = min(driver->max_speed, g->max_speed);
1325 ret = cdnsp_run(pdev, max_speed);
1327 spin_unlock_irqrestore(&pdev->lock, flags);
1333 * Update Event Ring Dequeue Pointer:
1334 * - When all events have finished
1335 * - To avoid "Event Ring Full Error" condition
1337 void cdnsp_update_erst_dequeue(struct cdnsp_device *pdev,
1338 union cdnsp_trb *event_ring_deq,
1344 temp_64 = cdnsp_read_64(&pdev->ir_set->erst_dequeue);
1346 /* If necessary, update the HW's version of the event ring deq ptr. */
1347 if (event_ring_deq != pdev->event_ring->dequeue) {
1348 deq = cdnsp_trb_virt_to_dma(pdev->event_ring->deq_seg,
1349 pdev->event_ring->dequeue);
1350 temp_64 &= ERST_PTR_MASK;
1351 temp_64 |= ((u64)deq & (u64)~ERST_PTR_MASK);
1354 /* Clear the event handler busy flag (RW1C). */
1356 temp_64 |= ERST_EHB;
1358 temp_64 &= ~ERST_EHB;
1360 cdnsp_write_64(temp_64, &pdev->ir_set->erst_dequeue);
1363 static void cdnsp_clear_cmd_ring(struct cdnsp_device *pdev)
1365 struct cdnsp_segment *seg;
1369 cdnsp_initialize_ring_info(pdev->cmd_ring);
1371 seg = pdev->cmd_ring->first_seg;
1372 for (i = 0; i < pdev->cmd_ring->num_segs; i++) {
1373 memset(seg->trbs, 0,
1374 sizeof(union cdnsp_trb) * (TRBS_PER_SEGMENT - 1));
1378 /* Set the address in the Command Ring Control register. */
1379 val_64 = cdnsp_read_64(&pdev->op_regs->cmd_ring);
1380 val_64 = (val_64 & (u64)CMD_RING_RSVD_BITS) |
1381 (pdev->cmd_ring->first_seg->dma & (u64)~CMD_RING_RSVD_BITS) |
1382 pdev->cmd_ring->cycle_state;
1383 cdnsp_write_64(val_64, &pdev->op_regs->cmd_ring);
1386 static void cdnsp_consume_all_events(struct cdnsp_device *pdev)
1388 struct cdnsp_segment *event_deq_seg;
1389 union cdnsp_trb *event_ring_deq;
1390 union cdnsp_trb *event;
1393 event_ring_deq = pdev->event_ring->dequeue;
1394 event_deq_seg = pdev->event_ring->deq_seg;
1395 event = pdev->event_ring->dequeue;
1397 /* Update ring dequeue pointer. */
1399 cycle_bit = (le32_to_cpu(event->event_cmd.flags) & TRB_CYCLE);
1401 /* Does the controller or driver own the TRB? */
1402 if (cycle_bit != pdev->event_ring->cycle_state)
1405 cdnsp_inc_deq(pdev, pdev->event_ring);
1407 if (!cdnsp_last_trb_on_seg(event_deq_seg, event)) {
1412 if (cdnsp_last_trb_on_ring(pdev->event_ring, event_deq_seg,
1416 event_deq_seg = event_deq_seg->next;
1417 event = event_deq_seg->trbs;
1420 cdnsp_update_erst_dequeue(pdev, event_ring_deq, 1);
1423 static void cdnsp_stop(struct cdnsp_device *pdev)
1427 cdnsp_cmd_flush_ep(pdev, &pdev->eps[0]);
1429 /* Remove internally queued request for ep0. */
1430 if (!list_empty(&pdev->eps[0].pending_list)) {
1431 struct cdnsp_request *req;
1433 req = next_request(&pdev->eps[0].pending_list);
1434 if (req == &pdev->ep0_preq)
1435 cdnsp_ep_dequeue(&pdev->eps[0], req);
1438 cdnsp_disable_port(pdev, &pdev->usb2_port.regs->portsc);
1439 cdnsp_disable_port(pdev, &pdev->usb3_port.regs->portsc);
1440 cdnsp_disable_slot(pdev);
1443 temp = readl(&pdev->op_regs->status);
1444 writel((temp & ~0x1fff) | STS_EINT, &pdev->op_regs->status);
1445 temp = readl(&pdev->ir_set->irq_pending);
1446 writel(IMAN_IE_CLEAR(temp), &pdev->ir_set->irq_pending);
1448 cdnsp_clear_port_change_bit(pdev, &pdev->usb2_port.regs->portsc);
1449 cdnsp_clear_port_change_bit(pdev, &pdev->usb3_port.regs->portsc);
1451 /* Clear interrupt line */
1452 temp = readl(&pdev->ir_set->irq_pending);
1454 writel(temp, &pdev->ir_set->irq_pending);
1456 cdnsp_consume_all_events(pdev);
1457 cdnsp_clear_cmd_ring(pdev);
1459 trace_cdnsp_exit("Controller stopped.");
1464 * This function is called by the gadget core when the driver is removed.
1465 * Disable slot, disable IRQs, and quiesce the controller.
1467 static int cdnsp_gadget_udc_stop(struct usb_gadget *g)
1469 struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1470 unsigned long flags;
1472 spin_lock_irqsave(&pdev->lock, flags);
1474 pdev->gadget_driver = NULL;
1475 spin_unlock_irqrestore(&pdev->lock, flags);
1480 static int cdnsp_gadget_get_frame(struct usb_gadget *g)
1482 struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1484 return cdnsp_get_frame(pdev);
1487 static void __cdnsp_gadget_wakeup(struct cdnsp_device *pdev)
1489 struct cdnsp_port_regs __iomem *port_regs;
1492 port_regs = pdev->active_port->regs;
1493 portsc = readl(&port_regs->portsc) & PORT_PLS_MASK;
1495 /* Remote wakeup feature is not enabled by host. */
1496 if (pdev->gadget.speed < USB_SPEED_SUPER && portsc == XDEV_U2) {
1497 portpm = readl(&port_regs->portpmsc);
1499 if (!(portpm & PORT_RWE))
1503 if (portsc == XDEV_U3 && !pdev->may_wakeup)
1506 cdnsp_set_link_state(pdev, &port_regs->portsc, XDEV_U0);
1508 pdev->cdnsp_state |= CDNSP_WAKEUP_PENDING;
1511 static int cdnsp_gadget_wakeup(struct usb_gadget *g)
1513 struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1514 unsigned long flags;
1516 spin_lock_irqsave(&pdev->lock, flags);
1517 __cdnsp_gadget_wakeup(pdev);
1518 spin_unlock_irqrestore(&pdev->lock, flags);
1523 static int cdnsp_gadget_set_selfpowered(struct usb_gadget *g,
1526 struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1527 unsigned long flags;
1529 spin_lock_irqsave(&pdev->lock, flags);
1530 g->is_selfpowered = !!is_selfpowered;
1531 spin_unlock_irqrestore(&pdev->lock, flags);
1536 static int cdnsp_gadget_pullup(struct usb_gadget *gadget, int is_on)
1538 struct cdnsp_device *pdev = gadget_to_cdnsp(gadget);
1539 struct cdns *cdns = dev_get_drvdata(pdev->dev);
1540 unsigned long flags;
1542 trace_cdnsp_pullup(is_on);
1545 * Disable events handling while controller is being
1548 disable_irq(cdns->dev_irq);
1549 spin_lock_irqsave(&pdev->lock, flags);
1552 cdnsp_reset_device(pdev);
1553 cdns_clear_vbus(cdns);
1555 cdns_set_vbus(cdns);
1558 spin_unlock_irqrestore(&pdev->lock, flags);
1559 enable_irq(cdns->dev_irq);
1564 static const struct usb_gadget_ops cdnsp_gadget_ops = {
1565 .get_frame = cdnsp_gadget_get_frame,
1566 .wakeup = cdnsp_gadget_wakeup,
1567 .set_selfpowered = cdnsp_gadget_set_selfpowered,
1568 .pullup = cdnsp_gadget_pullup,
1569 .udc_start = cdnsp_gadget_udc_start,
1570 .udc_stop = cdnsp_gadget_udc_stop,
1573 static void cdnsp_get_ep_buffering(struct cdnsp_device *pdev,
1574 struct cdnsp_ep *pep)
1576 void __iomem *reg = &pdev->cap_regs->hc_capbase;
1579 reg += cdnsp_find_next_ext_cap(reg, 0, XBUF_CAP_ID);
1581 if (!pep->direction) {
1582 pep->buffering = readl(reg + XBUF_RX_TAG_MASK_0_OFFSET);
1583 pep->buffering_period = readl(reg + XBUF_RX_TAG_MASK_1_OFFSET);
1584 pep->buffering = (pep->buffering + 1) / 2;
1585 pep->buffering_period = (pep->buffering_period + 1) / 2;
1589 endpoints = HCS_ENDPOINTS(pdev->hcs_params1) / 2;
1591 /* Set to XBUF_TX_TAG_MASK_0 register. */
1592 reg += XBUF_TX_CMD_OFFSET + (endpoints * 2 + 2) * sizeof(u32);
1593 /* Set reg to XBUF_TX_TAG_MASK_N related with this endpoint. */
1594 reg += pep->number * sizeof(u32) * 2;
1596 pep->buffering = (readl(reg) + 1) / 2;
1597 pep->buffering_period = pep->buffering;
1600 static int cdnsp_gadget_init_endpoints(struct cdnsp_device *pdev)
1602 int max_streams = HCC_MAX_PSA(pdev->hcc_params);
1603 struct cdnsp_ep *pep;
1606 INIT_LIST_HEAD(&pdev->gadget.ep_list);
1608 if (max_streams < STREAM_LOG_STREAMS) {
1609 dev_err(pdev->dev, "Stream size %d not supported\n",
1614 max_streams = STREAM_LOG_STREAMS;
1616 for (i = 0; i < CDNSP_ENDPOINTS_NUM; i++) {
1617 bool direction = !(i & 1); /* Start from OUT endpoint. */
1618 u8 epnum = ((i + 1) >> 1);
1620 if (!CDNSP_IF_EP_EXIST(pdev, epnum, direction))
1623 pep = &pdev->eps[i];
1625 pep->number = epnum;
1626 pep->direction = direction; /* 0 for OUT, 1 for IN. */
1629 * Ep0 is bidirectional, so ep0in and ep0out are represented by
1633 snprintf(pep->name, sizeof(pep->name), "ep%d%s",
1637 usb_ep_set_maxpacket_limit(&pep->endpoint, 512);
1638 pep->endpoint.maxburst = 1;
1639 pep->endpoint.ops = &cdnsp_gadget_ep0_ops;
1640 pep->endpoint.desc = &cdnsp_gadget_ep0_desc;
1641 pep->endpoint.comp_desc = NULL;
1642 pep->endpoint.caps.type_control = true;
1643 pep->endpoint.caps.dir_in = true;
1644 pep->endpoint.caps.dir_out = true;
1646 pdev->ep0_preq.epnum = pep->number;
1647 pdev->ep0_preq.pep = pep;
1648 pdev->gadget.ep0 = &pep->endpoint;
1650 snprintf(pep->name, sizeof(pep->name), "ep%d%s",
1651 epnum, (pep->direction) ? "in" : "out");
1653 pep->idx = (epnum * 2 + (direction ? 1 : 0)) - 1;
1654 usb_ep_set_maxpacket_limit(&pep->endpoint, 1024);
1656 pep->endpoint.max_streams = max_streams;
1657 pep->endpoint.ops = &cdnsp_gadget_ep_ops;
1658 list_add_tail(&pep->endpoint.ep_list,
1659 &pdev->gadget.ep_list);
1661 pep->endpoint.caps.type_iso = true;
1662 pep->endpoint.caps.type_bulk = true;
1663 pep->endpoint.caps.type_int = true;
1665 pep->endpoint.caps.dir_in = direction;
1666 pep->endpoint.caps.dir_out = !direction;
1669 pep->endpoint.name = pep->name;
1670 pep->in_ctx = cdnsp_get_ep_ctx(&pdev->in_ctx, pep->idx);
1671 pep->out_ctx = cdnsp_get_ep_ctx(&pdev->out_ctx, pep->idx);
1672 cdnsp_get_ep_buffering(pdev, pep);
1674 dev_dbg(pdev->dev, "Init %s, MPS: %04x SupType: "
1675 "CTRL: %s, INT: %s, BULK: %s, ISOC %s, "
1676 "SupDir IN: %s, OUT: %s\n",
1678 (pep->endpoint.caps.type_control) ? "yes" : "no",
1679 (pep->endpoint.caps.type_int) ? "yes" : "no",
1680 (pep->endpoint.caps.type_bulk) ? "yes" : "no",
1681 (pep->endpoint.caps.type_iso) ? "yes" : "no",
1682 (pep->endpoint.caps.dir_in) ? "yes" : "no",
1683 (pep->endpoint.caps.dir_out) ? "yes" : "no");
1685 INIT_LIST_HEAD(&pep->pending_list);
1691 static void cdnsp_gadget_free_endpoints(struct cdnsp_device *pdev)
1693 struct cdnsp_ep *pep;
1696 for (i = 0; i < CDNSP_ENDPOINTS_NUM; i++) {
1697 pep = &pdev->eps[i];
1698 if (pep->number != 0 && pep->out_ctx)
1699 list_del(&pep->endpoint.ep_list);
1703 void cdnsp_disconnect_gadget(struct cdnsp_device *pdev)
1705 pdev->cdnsp_state |= CDNSP_STATE_DISCONNECT_PENDING;
1707 if (pdev->gadget_driver && pdev->gadget_driver->disconnect) {
1708 spin_unlock(&pdev->lock);
1709 pdev->gadget_driver->disconnect(&pdev->gadget);
1710 spin_lock(&pdev->lock);
1713 pdev->gadget.speed = USB_SPEED_UNKNOWN;
1714 usb_gadget_set_state(&pdev->gadget, USB_STATE_NOTATTACHED);
1716 pdev->cdnsp_state &= ~CDNSP_STATE_DISCONNECT_PENDING;
1719 void cdnsp_suspend_gadget(struct cdnsp_device *pdev)
1721 if (pdev->gadget_driver && pdev->gadget_driver->suspend) {
1722 spin_unlock(&pdev->lock);
1723 pdev->gadget_driver->suspend(&pdev->gadget);
1724 spin_lock(&pdev->lock);
1728 void cdnsp_resume_gadget(struct cdnsp_device *pdev)
1730 if (pdev->gadget_driver && pdev->gadget_driver->resume) {
1731 spin_unlock(&pdev->lock);
1732 pdev->gadget_driver->resume(&pdev->gadget);
1733 spin_lock(&pdev->lock);
1737 void cdnsp_irq_reset(struct cdnsp_device *pdev)
1739 struct cdnsp_port_regs __iomem *port_regs;
1741 cdnsp_reset_device(pdev);
1743 port_regs = pdev->active_port->regs;
1744 pdev->gadget.speed = cdnsp_port_speed(readl(port_regs));
1746 spin_unlock(&pdev->lock);
1747 usb_gadget_udc_reset(&pdev->gadget, pdev->gadget_driver);
1748 spin_lock(&pdev->lock);
1750 switch (pdev->gadget.speed) {
1751 case USB_SPEED_SUPER_PLUS:
1752 case USB_SPEED_SUPER:
1753 cdnsp_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1754 pdev->gadget.ep0->maxpacket = 512;
1756 case USB_SPEED_HIGH:
1757 case USB_SPEED_FULL:
1758 cdnsp_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
1759 pdev->gadget.ep0->maxpacket = 64;
1762 /* Low speed is not supported. */
1763 dev_err(pdev->dev, "Unknown device speed\n");
1767 cdnsp_clear_chicken_bits_2(pdev, CHICKEN_XDMA_2_TP_CACHE_DIS);
1768 cdnsp_setup_device(pdev, SETUP_CONTEXT_ONLY);
1769 usb_gadget_set_state(&pdev->gadget, USB_STATE_DEFAULT);
1772 static void cdnsp_get_rev_cap(struct cdnsp_device *pdev)
1774 void __iomem *reg = &pdev->cap_regs->hc_capbase;
1776 reg += cdnsp_find_next_ext_cap(reg, 0, RTL_REV_CAP);
1777 pdev->rev_cap = reg;
1779 dev_info(pdev->dev, "Rev: %08x/%08x, eps: %08x, buff: %08x/%08x\n",
1780 readl(&pdev->rev_cap->ctrl_revision),
1781 readl(&pdev->rev_cap->rtl_revision),
1782 readl(&pdev->rev_cap->ep_supported),
1783 readl(&pdev->rev_cap->rx_buff_size),
1784 readl(&pdev->rev_cap->tx_buff_size));
1787 static int cdnsp_gen_setup(struct cdnsp_device *pdev)
1792 pdev->cap_regs = pdev->regs;
1793 pdev->op_regs = pdev->regs +
1794 HC_LENGTH(readl(&pdev->cap_regs->hc_capbase));
1795 pdev->run_regs = pdev->regs +
1796 (readl(&pdev->cap_regs->run_regs_off) & RTSOFF_MASK);
1798 /* Cache read-only capability registers */
1799 pdev->hcs_params1 = readl(&pdev->cap_regs->hcs_params1);
1800 pdev->hcc_params = readl(&pdev->cap_regs->hc_capbase);
1801 pdev->hci_version = HC_VERSION(pdev->hcc_params);
1802 pdev->hcc_params = readl(&pdev->cap_regs->hcc_params);
1804 cdnsp_get_rev_cap(pdev);
1806 /* Make sure the Device Controller is halted. */
1807 ret = cdnsp_halt(pdev);
1811 /* Reset the internal controller memory state and registers. */
1812 ret = cdnsp_reset(pdev);
1817 * Set dma_mask and coherent_dma_mask to 64-bits,
1818 * if controller supports 64-bit addressing.
1820 if (HCC_64BIT_ADDR(pdev->hcc_params) &&
1821 !dma_set_mask(pdev->dev, DMA_BIT_MASK(64))) {
1822 dev_dbg(pdev->dev, "Enabling 64-bit DMA addresses.\n");
1823 dma_set_coherent_mask(pdev->dev, DMA_BIT_MASK(64));
1826 * This is to avoid error in cases where a 32-bit USB
1827 * controller is used on a 64-bit capable system.
1829 ret = dma_set_mask(pdev->dev, DMA_BIT_MASK(32));
1833 dev_dbg(pdev->dev, "Enabling 32-bit DMA addresses.\n");
1834 dma_set_coherent_mask(pdev->dev, DMA_BIT_MASK(32));
1837 spin_lock_init(&pdev->lock);
1839 ret = cdnsp_mem_init(pdev);
1844 * Software workaround for U1: after transition
1845 * to U1 the controller starts gating clock, and in some cases,
1846 * it causes that controller stack.
1848 reg = readl(&pdev->port3x_regs->mode_2);
1849 reg &= ~CFG_3XPORT_U1_PIPE_CLK_GATE_EN;
1850 writel(reg, &pdev->port3x_regs->mode_2);
1855 static int __cdnsp_gadget_init(struct cdns *cdns)
1857 struct cdnsp_device *pdev;
1861 cdns_drd_gadget_on(cdns);
1863 pdev = kzalloc(sizeof(*pdev), GFP_KERNEL);
1867 pm_runtime_get_sync(cdns->dev);
1869 cdns->gadget_dev = pdev;
1870 pdev->dev = cdns->dev;
1871 pdev->regs = cdns->dev_regs;
1872 max_speed = usb_get_maximum_speed(cdns->dev);
1874 switch (max_speed) {
1875 case USB_SPEED_FULL:
1876 case USB_SPEED_HIGH:
1877 case USB_SPEED_SUPER:
1878 case USB_SPEED_SUPER_PLUS:
1881 dev_err(cdns->dev, "invalid speed parameter %d\n", max_speed);
1883 case USB_SPEED_UNKNOWN:
1884 /* Default to SSP */
1885 max_speed = USB_SPEED_SUPER_PLUS;
1889 pdev->gadget.ops = &cdnsp_gadget_ops;
1890 pdev->gadget.name = "cdnsp-gadget";
1891 pdev->gadget.speed = USB_SPEED_UNKNOWN;
1892 pdev->gadget.sg_supported = 1;
1893 pdev->gadget.max_speed = max_speed;
1894 pdev->gadget.lpm_capable = 1;
1896 pdev->setup_buf = kzalloc(CDNSP_EP0_SETUP_SIZE, GFP_KERNEL);
1897 if (!pdev->setup_buf)
1901 * Controller supports not aligned buffer but it should improve
1904 pdev->gadget.quirk_ep_out_aligned_size = true;
1906 ret = cdnsp_gen_setup(pdev);
1908 dev_err(pdev->dev, "Generic initialization failed %d\n", ret);
1912 ret = cdnsp_gadget_init_endpoints(pdev);
1914 dev_err(pdev->dev, "failed to initialize endpoints\n");
1918 ret = usb_add_gadget_udc(pdev->dev, &pdev->gadget);
1920 dev_err(pdev->dev, "failed to register udc\n");
1921 goto free_endpoints;
1924 ret = devm_request_threaded_irq(pdev->dev, cdns->dev_irq,
1926 cdnsp_thread_irq_handler, IRQF_SHARED,
1927 dev_name(pdev->dev), pdev);
1934 usb_del_gadget_udc(&pdev->gadget);
1936 cdnsp_gadget_free_endpoints(pdev);
1940 cdnsp_mem_cleanup(pdev);
1942 kfree(pdev->setup_buf);
1949 static void cdnsp_gadget_exit(struct cdns *cdns)
1951 struct cdnsp_device *pdev = cdns->gadget_dev;
1953 devm_free_irq(pdev->dev, cdns->dev_irq, pdev);
1954 pm_runtime_mark_last_busy(cdns->dev);
1955 pm_runtime_put_autosuspend(cdns->dev);
1956 usb_del_gadget_udc(&pdev->gadget);
1957 cdnsp_gadget_free_endpoints(pdev);
1958 cdnsp_mem_cleanup(pdev);
1960 cdns->gadget_dev = NULL;
1961 cdns_drd_gadget_off(cdns);
1964 static int cdnsp_gadget_suspend(struct cdns *cdns, bool do_wakeup)
1966 struct cdnsp_device *pdev = cdns->gadget_dev;
1967 unsigned long flags;
1969 if (pdev->link_state == XDEV_U3)
1972 spin_lock_irqsave(&pdev->lock, flags);
1973 cdnsp_disconnect_gadget(pdev);
1975 spin_unlock_irqrestore(&pdev->lock, flags);
1980 static int cdnsp_gadget_resume(struct cdns *cdns, bool hibernated)
1982 struct cdnsp_device *pdev = cdns->gadget_dev;
1983 enum usb_device_speed max_speed;
1984 unsigned long flags;
1987 if (!pdev->gadget_driver)
1990 spin_lock_irqsave(&pdev->lock, flags);
1991 max_speed = pdev->gadget_driver->max_speed;
1993 /* Limit speed if necessary. */
1994 max_speed = min(max_speed, pdev->gadget.max_speed);
1996 ret = cdnsp_run(pdev, max_speed);
1998 if (pdev->link_state == XDEV_U3)
1999 __cdnsp_gadget_wakeup(pdev);
2001 spin_unlock_irqrestore(&pdev->lock, flags);
2007 * cdnsp_gadget_init - initialize device structure
2008 * @cdns: cdnsp instance
2010 * This function initializes the gadget.
2012 int cdnsp_gadget_init(struct cdns *cdns)
2014 struct cdns_role_driver *rdrv;
2016 rdrv = devm_kzalloc(cdns->dev, sizeof(*rdrv), GFP_KERNEL);
2020 rdrv->start = __cdnsp_gadget_init;
2021 rdrv->stop = cdnsp_gadget_exit;
2022 rdrv->suspend = cdnsp_gadget_suspend;
2023 rdrv->resume = cdnsp_gadget_resume;
2024 rdrv->state = CDNS_ROLE_STATE_INACTIVE;
2025 rdrv->name = "gadget";
2026 cdns->roles[USB_ROLE_DEVICE] = rdrv;