drivers/usb/dwc3/gadget.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
   4  *
   5  * Copyright (C) 2010-2011 Texas Instruments Incorporated - http://www.ti.com
   6  *
   7  * Authors: Felipe Balbi <balbi@ti.com>,
   8  *          Sebastian Andrzej Siewior <bigeasy@linutronix.de>
   9  */
  10
  11 #include <linux/kernel.h>
  12 #include <linux/delay.h>
  13 #include <linux/slab.h>
  14 #include <linux/spinlock.h>
  15 #include <linux/platform_device.h>
  16 #include <linux/pm_runtime.h>
  17 #include <linux/interrupt.h>
  18 #include <linux/io.h>
  19 #include <linux/list.h>
  20 #include <linux/dma-mapping.h>
  21
  22 #include <linux/usb/ch9.h>
  23 #include <linux/usb/gadget.h>
  24
  25 #include "debug.h"
  26 #include "core.h"
  27 #include "gadget.h"
  28 #include "io.h"
  29
  30 #define DWC3_ALIGN_FRAME(d, n)  (((d)->frame_number + ((d)->interval * (n))) \
  31                                         & ~((d)->interval - 1))
  32
  33 /**
  34  * dwc3_gadget_set_test_mode - enables usb2 test modes
  35  * @dwc: pointer to our context structure
  36  * @mode: the mode to set (J, K SE0 NAK, Force Enable)
  37  *
  38  * Caller should take care of locking. This function will return 0 on
  39  * success or -EINVAL if wrong Test Selector is passed.
  40  */
  41 int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode)
  42 {
  43         u32             reg;
  44
  45         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
  46         reg &= ~DWC3_DCTL_TSTCTRL_MASK;
  47
  48         switch (mode) {
  49         case TEST_J:
  50         case TEST_K:
  51         case TEST_SE0_NAK:
  52         case TEST_PACKET:
  53         case TEST_FORCE_EN:
  54                 reg |= mode << 1;
  55                 break;
  56         default:
  57                 return -EINVAL;
  58         }
  59
  60         dwc3_gadget_dctl_write_safe(dwc, reg);
  61
  62         return 0;
  63 }
  64
  65 /**
  66  * dwc3_gadget_get_link_state - gets current state of usb link
  67  * @dwc: pointer to our context structure
  68  *
  69  * Caller should take care of locking. This function will
  70  * return the link state on success (>= 0) or -ETIMEDOUT.
  71  */
  72 int dwc3_gadget_get_link_state(struct dwc3 *dwc)
  73 {
  74         u32             reg;
  75
  76         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
  77
  78         return DWC3_DSTS_USBLNKST(reg);
  79 }
  80
  81 /**
  82  * dwc3_gadget_set_link_state - sets usb link to a particular state
  83  * @dwc: pointer to our context structure
  84  * @state: the state to put link into
  85  *
  86  * Caller should take care of locking. This function will
  87  * return 0 on success or -ETIMEDOUT.
  88  */
  89 int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state)
  90 {
  91         int             retries = 10000;
  92         u32             reg;
  93
  94         /*
  95          * Wait until device controller is ready. Only applies to 1.94a and
  96          * later RTL.
  97          */
  98         if (dwc->revision >= DWC3_REVISION_194A) {
  99                 while (--retries) {
 100                         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
 101                         if (reg & DWC3_DSTS_DCNRD)
 102                                 udelay(5);
 103                         else
 104                                 break;
 105                 }
 106
 107                 if (retries <= 0)
 108                         return -ETIMEDOUT;
 109         }
 110
 111         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
 112         reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
 113
 114         /* set no action before sending new link state change */
 115         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
 116
 117         /* set requested state */
 118         reg |= DWC3_DCTL_ULSTCHNGREQ(state);
 119         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
 120
 121         /*
 122          * The following code is racy when called from dwc3_gadget_wakeup,
 123          * and is not needed, at least on newer versions
 124          */
 125         if (dwc->revision >= DWC3_REVISION_194A)
 126                 return 0;
 127
 128         /* wait for a change in DSTS */
 129         retries = 10000;
 130         while (--retries) {
 131                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
 132
 133                 if (DWC3_DSTS_USBLNKST(reg) == state)
 134                         return 0;
 135
 136                 udelay(5);
 137         }
 138
 139         return -ETIMEDOUT;
 140 }
 141
 142 /**
 143  * dwc3_ep_inc_trb - increment a trb index.
 144  * @index: Pointer to the TRB index to increment.
 145  *
 146  * The index should never point to the link TRB. After incrementing,
 147  * if it is point to the link TRB, wrap around to the beginning. The
 148  * link TRB is always at the last TRB entry.
 149  */
 150 static void dwc3_ep_inc_trb(u8 *index)
 151 {
 152         (*index)++;
 153         if (*index == (DWC3_TRB_NUM - 1))
 154                 *index = 0;
 155 }
 156
 157 /**
 158  * dwc3_ep_inc_enq - increment endpoint's enqueue pointer
 159  * @dep: The endpoint whose enqueue pointer we're incrementing
 160  */
 161 static void dwc3_ep_inc_enq(struct dwc3_ep *dep)
 162 {
 163         dwc3_ep_inc_trb(&dep->trb_enqueue);
 164 }
 165
 166 /**
 167  * dwc3_ep_inc_deq - increment endpoint's dequeue pointer
 168  * @dep: The endpoint whose enqueue pointer we're incrementing
 169  */
 170 static void dwc3_ep_inc_deq(struct dwc3_ep *dep)
 171 {
 172         dwc3_ep_inc_trb(&dep->trb_dequeue);
 173 }
 174
 175 static void dwc3_gadget_del_and_unmap_request(struct dwc3_ep *dep,
 176                 struct dwc3_request *req, int status)
 177 {
 178         struct dwc3                     *dwc = dep->dwc;
 179
 180         list_del(&req->list);
 181         req->remaining = 0;
 182         req->needs_extra_trb = false;
 183
 184         if (req->request.status == -EINPROGRESS)
 185                 req->request.status = status;
 186
 187         if (req->trb)
 188                 usb_gadget_unmap_request_by_dev(dwc->sysdev,
 189                                 &req->request, req->direction);
 190
 191         req->trb = NULL;
 192         trace_dwc3_gadget_giveback(req);
 193
 194         if (dep->number > 1)
 195                 pm_runtime_put(dwc->dev);
 196 }
 197
 198 /**
 199  * dwc3_gadget_giveback - call struct usb_request's ->complete callback
 200  * @dep: The endpoint to whom the request belongs to
 201  * @req: The request we're giving back
 202  * @status: completion code for the request
 203  *
 204  * Must be called with controller's lock held and interrupts disabled. This
 205  * function will unmap @req and call its ->complete() callback to notify upper
 206  * layers that it has completed.
 207  */
 208 void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
 209                 int status)
 210 {
 211         struct dwc3                     *dwc = dep->dwc;
 212
 213         dwc3_gadget_del_and_unmap_request(dep, req, status);
 214         req->status = DWC3_REQUEST_STATUS_COMPLETED;
 215
 216         spin_unlock(&dwc->lock);
 217         usb_gadget_giveback_request(&dep->endpoint, &req->request);
 218         spin_lock(&dwc->lock);
 219 }
 220
 221 /**
 222  * dwc3_send_gadget_generic_command - issue a generic command for the controller
 223  * @dwc: pointer to the controller context
 224  * @cmd: the command to be issued
 225  * @param: command parameter
 226  *
 227  * Caller should take care of locking. Issue @cmd with a given @param to @dwc
 228  * and wait for its completion.
 229  */
 230 int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned cmd, u32 param)
 231 {
 232         u32             timeout = 500;
 233         int             status = 0;
 234         int             ret = 0;
 235         u32             reg;
 236
 237         dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param);
 238         dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT);
 239
 240         do {
 241                 reg = dwc3_readl(dwc->regs, DWC3_DGCMD);
 242                 if (!(reg & DWC3_DGCMD_CMDACT)) {
 243                         status = DWC3_DGCMD_STATUS(reg);
 244                         if (status)
 245                                 ret = -EINVAL;
 246                         break;
 247                 }
 248         } while (--timeout);
 249
 250         if (!timeout) {
 251                 ret = -ETIMEDOUT;
 252                 status = -ETIMEDOUT;
 253         }
 254
 255         trace_dwc3_gadget_generic_cmd(cmd, param, status);
 256
 257         return ret;
 258 }
 259
 260 static int __dwc3_gadget_wakeup(struct dwc3 *dwc);
 261
 262 /**
 263  * dwc3_send_gadget_ep_cmd - issue an endpoint command
 264  * @dep: the endpoint to which the command is going to be issued
 265  * @cmd: the command to be issued
 266  * @params: parameters to the command
 267  *
 268  * Caller should handle locking. This function will issue @cmd with given
 269  * @params to @dep and wait for its completion.
 270  */
 271 int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned cmd,
 272                 struct dwc3_gadget_ep_cmd_params *params)
 273 {
 274         const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
 275         struct dwc3             *dwc = dep->dwc;
 276         u32                     timeout = 1000;
 277         u32                     saved_config = 0;
 278         u32                     reg;
 279
 280         int                     cmd_status = 0;
 281         int                     ret = -EINVAL;
 282
 283         /*
 284          * When operating in USB 2.0 speeds (HS/FS), if GUSB2PHYCFG.ENBLSLPM or
 285          * GUSB2PHYCFG.SUSPHY is set, it must be cleared before issuing an
 286          * endpoint command.
 287          *
 288          * Save and clear both GUSB2PHYCFG.ENBLSLPM and GUSB2PHYCFG.SUSPHY
 289          * settings. Restore them after the command is completed.
 290          *
 291          * DWC_usb3 3.30a and DWC_usb31 1.90a programming guide section 3.2.2
 292          */
 293         if (dwc->gadget.speed <= USB_SPEED_HIGH) {
 294                 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
 295                 if (unlikely(reg & DWC3_GUSB2PHYCFG_SUSPHY)) {
 296                         saved_config |= DWC3_GUSB2PHYCFG_SUSPHY;
 297                         reg &= ~DWC3_GUSB2PHYCFG_SUSPHY;
 298                 }
 299
 300                 if (reg & DWC3_GUSB2PHYCFG_ENBLSLPM) {
 301                         saved_config |= DWC3_GUSB2PHYCFG_ENBLSLPM;
 302                         reg &= ~DWC3_GUSB2PHYCFG_ENBLSLPM;
 303                 }
 304
 305                 if (saved_config)
 306                         dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
 307         }
 308
 309         if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
 310                 int             needs_wakeup;
 311
 312                 needs_wakeup = (dwc->link_state == DWC3_LINK_STATE_U1 ||
 313                                 dwc->link_state == DWC3_LINK_STATE_U2 ||
 314                                 dwc->link_state == DWC3_LINK_STATE_U3);
 315
 316                 if (unlikely(needs_wakeup)) {
 317                         ret = __dwc3_gadget_wakeup(dwc);
 318                         dev_WARN_ONCE(dwc->dev, ret, "wakeup failed --> %d\n",
 319                                         ret);
 320                 }
 321         }
 322
 323         dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0);
 324         dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1);
 325         dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2);
 326
 327         /*
 328          * Synopsys Databook 2.60a states in section 6.3.2.5.6 of that if we're
 329          * not relying on XferNotReady, we can make use of a special "No
 330          * Response Update Transfer" command where we should clear both CmdAct
 331          * and CmdIOC bits.
 332          *
 333          * With this, we don't need to wait for command completion and can
 334          * straight away issue further commands to the endpoint.
 335          *
 336          * NOTICE: We're making an assumption that control endpoints will never
 337          * make use of Update Transfer command. This is a safe assumption
 338          * because we can never have more than one request at a time with
 339          * Control Endpoints. If anybody changes that assumption, this chunk
 340          * needs to be updated accordingly.
 341          */
 342         if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_UPDATETRANSFER &&
 343                         !usb_endpoint_xfer_isoc(desc))
 344                 cmd &= ~(DWC3_DEPCMD_CMDIOC | DWC3_DEPCMD_CMDACT);
 345         else
 346                 cmd |= DWC3_DEPCMD_CMDACT;
 347
 348         dwc3_writel(dep->regs, DWC3_DEPCMD, cmd);
 349         do {
 350                 reg = dwc3_readl(dep->regs, DWC3_DEPCMD);
 351                 if (!(reg & DWC3_DEPCMD_CMDACT)) {
 352                         cmd_status = DWC3_DEPCMD_STATUS(reg);
 353
 354                         switch (cmd_status) {
 355                         case 0:
 356                                 ret = 0;
 357                                 break;
 358                         case DEPEVT_TRANSFER_NO_RESOURCE:
 359                                 ret = -EINVAL;
 360                                 break;
 361                         case DEPEVT_TRANSFER_BUS_EXPIRY:
 362                                 /*
 363                                  * SW issues START TRANSFER command to
 364                                  * isochronous ep with future frame interval. If
 365                                  * future interval time has already passed when
 366                                  * core receives the command, it will respond
 367                                  * with an error status of 'Bus Expiry'.
 368                                  *
 369                                  * Instead of always returning -EINVAL, let's
 370                                  * give a hint to the gadget driver that this is
 371                                  * the case by returning -EAGAIN.
 372                                  */
 373                                 ret = -EAGAIN;
 374                                 break;
 375                         default:
 376                                 dev_WARN(dwc->dev, "UNKNOWN cmd status\n");
 377                         }
 378
 379                         break;
 380                 }
 381         } while (--timeout);
 382
 383         if (timeout == 0) {
 384                 ret = -ETIMEDOUT;
 385                 cmd_status = -ETIMEDOUT;
 386         }
 387
 388         trace_dwc3_gadget_ep_cmd(dep, cmd, params, cmd_status);
 389
 390         if (ret == 0 && DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
 391                 dep->flags |= DWC3_EP_TRANSFER_STARTED;
 392                 dwc3_gadget_ep_get_transfer_index(dep);
 393         }
 394
 395         if (saved_config) {
 396                 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
 397                 reg |= saved_config;
 398                 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
 399         }
 400
 401         return ret;
 402 }
 403
 404 static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep *dep)
 405 {
 406         struct dwc3 *dwc = dep->dwc;
 407         struct dwc3_gadget_ep_cmd_params params;
 408         u32 cmd = DWC3_DEPCMD_CLEARSTALL;
 409
 410         /*
 411          * As of core revision 2.60a the recommended programming model
 412          * is to set the ClearPendIN bit when issuing a Clear Stall EP
 413          * command for IN endpoints. This is to prevent an issue where
 414          * some (non-compliant) hosts may not send ACK TPs for pending
 415          * IN transfers due to a mishandled error condition. Synopsys
 416          * STAR 9000614252.
 417          */
 418         if (dep->direction && (dwc->revision >= DWC3_REVISION_260A) &&
 419             (dwc->gadget.speed >= USB_SPEED_SUPER))
 420                 cmd |= DWC3_DEPCMD_CLEARPENDIN;
 421
 422         memset(&params, 0, sizeof(params));
 423
 424         return dwc3_send_gadget_ep_cmd(dep, cmd, &params);
 425 }
 426
 427 static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep,
 428                 struct dwc3_trb *trb)
 429 {
 430         u32             offset = (char *) trb - (char *) dep->trb_pool;
 431
 432         return dep->trb_pool_dma + offset;
 433 }
 434
 435 static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)
 436 {
 437         struct dwc3             *dwc = dep->dwc;
 438
 439         if (dep->trb_pool)
 440                 return 0;
 441
 442         dep->trb_pool = dma_alloc_coherent(dwc->sysdev,
 443                         sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
 444                         &dep->trb_pool_dma, GFP_KERNEL);
 445         if (!dep->trb_pool) {
 446                 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",
 447                                 dep->name);
 448                 return -ENOMEM;
 449         }
 450
 451         return 0;
 452 }
 453
 454 static void dwc3_free_trb_pool(struct dwc3_ep *dep)
 455 {
 456         struct dwc3             *dwc = dep->dwc;
 457
 458         dma_free_coherent(dwc->sysdev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
 459                         dep->trb_pool, dep->trb_pool_dma);
 460
 461         dep->trb_pool = NULL;
 462         dep->trb_pool_dma = 0;
 463 }
 464
 465 static int dwc3_gadget_set_xfer_resource(struct dwc3_ep *dep)
 466 {
 467         struct dwc3_gadget_ep_cmd_params params;
 468
 469         memset(&params, 0x00, sizeof(params));
 470
 471         params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1);
 472
 473         return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETTRANSFRESOURCE,
 474                         &params);
 475 }
 476
 477 /**
 478  * dwc3_gadget_start_config - configure ep resources
 479  * @dep: endpoint that is being enabled
 480  *
 481  * Issue a %DWC3_DEPCMD_DEPSTARTCFG command to @dep. After the command's
 482  * completion, it will set Transfer Resource for all available endpoints.
 483  *
 484  * The assignment of transfer resources cannot perfectly follow the data book
 485  * due to the fact that the controller driver does not have all knowledge of the
 486  * configuration in advance. It is given this information piecemeal by the
 487  * composite gadget framework after every SET_CONFIGURATION and
 488  * SET_INTERFACE. Trying to follow the databook programming model in this
 489  * scenario can cause errors. For two reasons:
 490  *
 491  * 1) The databook says to do %DWC3_DEPCMD_DEPSTARTCFG for every
 492  * %USB_REQ_SET_CONFIGURATION and %USB_REQ_SET_INTERFACE (8.1.5). This is
 493  * incorrect in the scenario of multiple interfaces.
 494  *
 495  * 2) The databook does not mention doing more %DWC3_DEPCMD_DEPXFERCFG for new
 496  * endpoint on alt setting (8.1.6).
 497  *
 498  * The following simplified method is used instead:
 499  *
 500  * All hardware endpoints can be assigned a transfer resource and this setting
 501  * will stay persistent until either a core reset or hibernation. So whenever we
 502  * do a %DWC3_DEPCMD_DEPSTARTCFG(0) we can go ahead and do
 503  * %DWC3_DEPCMD_DEPXFERCFG for every hardware endpoint as well. We are
 504  * guaranteed that there are as many transfer resources as endpoints.
 505  *
 506  * This function is called for each endpoint when it is being enabled but is
 507  * triggered only when called for EP0-out, which always happens first, and which
 508  * should only happen in one of the above conditions.
 509  */
 510 static int dwc3_gadget_start_config(struct dwc3_ep *dep)
 511 {
 512         struct dwc3_gadget_ep_cmd_params params;
 513         struct dwc3             *dwc;
 514         u32                     cmd;
 515         int                     i;
 516         int                     ret;
 517
 518         if (dep->number)
 519                 return 0;
 520
 521         memset(&params, 0x00, sizeof(params));
 522         cmd = DWC3_DEPCMD_DEPSTARTCFG;
 523         dwc = dep->dwc;
 524
 525         ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
 526         if (ret)
 527                 return ret;
 528
 529         for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
 530                 struct dwc3_ep *dep = dwc->eps[i];
 531
 532                 if (!dep)
 533                         continue;
 534
 535                 ret = dwc3_gadget_set_xfer_resource(dep);
 536                 if (ret)
 537                         return ret;
 538         }
 539
 540         return 0;
 541 }
 542
 543 static int dwc3_gadget_set_ep_config(struct dwc3_ep *dep, unsigned int action)
 544 {
 545         const struct usb_ss_ep_comp_descriptor *comp_desc;
 546         const struct usb_endpoint_descriptor *desc;
 547         struct dwc3_gadget_ep_cmd_params params;
 548         struct dwc3 *dwc = dep->dwc;
 549
 550         comp_desc = dep->endpoint.comp_desc;
 551         desc = dep->endpoint.desc;
 552
 553         memset(&params, 0x00, sizeof(params));
 554
 555         params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc))
 556                 | DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc));
 557
 558         /* Burst size is only needed in SuperSpeed mode */
 559         if (dwc->gadget.speed >= USB_SPEED_SUPER) {
 560                 u32 burst = dep->endpoint.maxburst;
 561                 params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst - 1);
 562         }
 563
 564         params.param0 |= action;
 565         if (action == DWC3_DEPCFG_ACTION_RESTORE)
 566                 params.param2 |= dep->saved_state;
 567
 568         if (usb_endpoint_xfer_control(desc))
 569                 params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN;
 570
 571         if (dep->number <= 1 || usb_endpoint_xfer_isoc(desc))
 572                 params.param1 |= DWC3_DEPCFG_XFER_NOT_READY_EN;
 573
 574         if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
 575                 params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE
 576                         | DWC3_DEPCFG_STREAM_EVENT_EN;
 577                 dep->stream_capable = true;
 578         }
 579
 580         if (!usb_endpoint_xfer_control(desc))
 581                 params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN;
 582
 583         /*
 584          * We are doing 1:1 mapping for endpoints, meaning
 585          * Physical Endpoints 2 maps to Logical Endpoint 2 and
 586          * so on. We consider the direction bit as part of the physical
 587          * endpoint number. So USB endpoint 0x81 is 0x03.
 588          */
 589         params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number);
 590
 591         /*
 592          * We must use the lower 16 TX FIFOs even though
 593          * HW might have more
 594          */
 595         if (dep->direction)
 596                 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1);
 597
 598         if (desc->bInterval) {
 599                 params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(desc->bInterval - 1);
 600                 dep->interval = 1 << (desc->bInterval - 1);
 601         }
 602
 603         return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETEPCONFIG, &params);
 604 }
 605
 606 /**
 607  * __dwc3_gadget_ep_enable - initializes a hw endpoint
 608  * @dep: endpoint to be initialized
 609  * @action: one of INIT, MODIFY or RESTORE
 610  *
 611  * Caller should take care of locking. Execute all necessary commands to
 612  * initialize a HW endpoint so it can be used by a gadget driver.
 613  */
 614 static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep, unsigned int action)
 615 {
 616         const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
 617         struct dwc3             *dwc = dep->dwc;
 618
 619         u32                     reg;
 620         int                     ret;
 621
 622         if (!(dep->flags & DWC3_EP_ENABLED)) {
 623                 ret = dwc3_gadget_start_config(dep);
 624                 if (ret)
 625                         return ret;
 626         }
 627
 628         ret = dwc3_gadget_set_ep_config(dep, action);
 629         if (ret)
 630                 return ret;
 631
 632         if (!(dep->flags & DWC3_EP_ENABLED)) {
 633                 struct dwc3_trb *trb_st_hw;
 634                 struct dwc3_trb *trb_link;
 635
 636                 dep->type = usb_endpoint_type(desc);
 637                 dep->flags |= DWC3_EP_ENABLED;
 638
 639                 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
 640                 reg |= DWC3_DALEPENA_EP(dep->number);
 641                 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
 642
 643                 if (usb_endpoint_xfer_control(desc))
 644                         goto out;
 645
 646                 /* Initialize the TRB ring */
 647                 dep->trb_dequeue = 0;
 648                 dep->trb_enqueue = 0;
 649                 memset(dep->trb_pool, 0,
 650                        sizeof(struct dwc3_trb) * DWC3_TRB_NUM);
 651
 652                 /* Link TRB. The HWO bit is never reset */
 653                 trb_st_hw = &dep->trb_pool[0];
 654
 655                 trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1];
 656                 trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
 657                 trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
 658                 trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB;
 659                 trb_link->ctrl |= DWC3_TRB_CTRL_HWO;
 660         }
 661
 662         /*
 663          * Issue StartTransfer here with no-op TRB so we can always rely on No
 664          * Response Update Transfer command.
 665          */
 666         if ((usb_endpoint_xfer_bulk(desc) && !dep->stream_capable) ||
 667                         usb_endpoint_xfer_int(desc)) {
 668                 struct dwc3_gadget_ep_cmd_params params;
 669                 struct dwc3_trb *trb;
 670                 dma_addr_t trb_dma;
 671                 u32 cmd;
 672
 673                 memset(&params, 0, sizeof(params));
 674                 trb = &dep->trb_pool[0];
 675                 trb_dma = dwc3_trb_dma_offset(dep, trb);
 676
 677                 params.param0 = upper_32_bits(trb_dma);
 678                 params.param1 = lower_32_bits(trb_dma);
 679
 680                 cmd = DWC3_DEPCMD_STARTTRANSFER;
 681
 682                 ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
 683                 if (ret < 0)
 684                         return ret;
 685         }
 686
 687 out:
 688         trace_dwc3_gadget_ep_enable(dep);
 689
 690         return 0;
 691 }
 692
 693 static void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force,
 694                 bool interrupt);
 695 static void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep)
 696 {
 697         struct dwc3_request             *req;
 698
 699         dwc3_stop_active_transfer(dep, true, false);
 700
 701         /* - giveback all requests to gadget driver */
 702         while (!list_empty(&dep->started_list)) {
 703                 req = next_request(&dep->started_list);
 704
 705                 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
 706         }
 707
 708         while (!list_empty(&dep->pending_list)) {
 709                 req = next_request(&dep->pending_list);
 710
 711                 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
 712         }
 713
 714         while (!list_empty(&dep->cancelled_list)) {
 715                 req = next_request(&dep->cancelled_list);
 716
 717                 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
 718         }
 719 }
 720
 721 /**
 722  * __dwc3_gadget_ep_disable - disables a hw endpoint
 723  * @dep: the endpoint to disable
 724  *
 725  * This function undoes what __dwc3_gadget_ep_enable did and also removes
 726  * requests which are currently being processed by the hardware and those which
 727  * are not yet scheduled.
 728  *
 729  * Caller should take care of locking.
 730  */
 731 static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
 732 {
 733         struct dwc3             *dwc = dep->dwc;
 734         u32                     reg;
 735
 736         trace_dwc3_gadget_ep_disable(dep);
 737
 738         dwc3_remove_requests(dwc, dep);
 739
 740         /* make sure HW endpoint isn't stalled */
 741         if (dep->flags & DWC3_EP_STALL)
 742                 __dwc3_gadget_ep_set_halt(dep, 0, false);
 743
 744         reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
 745         reg &= ~DWC3_DALEPENA_EP(dep->number);
 746         dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
 747
 748         dep->stream_capable = false;
 749         dep->type = 0;
 750         dep->flags = 0;
 751
 752         /* Clear out the ep descriptors for non-ep0 */
 753         if (dep->number > 1) {
 754                 dep->endpoint.comp_desc = NULL;
 755                 dep->endpoint.desc = NULL;
 756         }
 757
 758         return 0;
 759 }
 760
 761 /* -------------------------------------------------------------------------- */
 762
 763 static int dwc3_gadget_ep0_enable(struct usb_ep *ep,
 764                 const struct usb_endpoint_descriptor *desc)
 765 {
 766         return -EINVAL;
 767 }
 768
 769 static int dwc3_gadget_ep0_disable(struct usb_ep *ep)
 770 {
 771         return -EINVAL;
 772 }
 773
 774 /* -------------------------------------------------------------------------- */
 775
 776 static int dwc3_gadget_ep_enable(struct usb_ep *ep,
 777                 const struct usb_endpoint_descriptor *desc)
 778 {
 779         struct dwc3_ep                  *dep;
 780         struct dwc3                     *dwc;
 781         unsigned long                   flags;
 782         int                             ret;
 783
 784         if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
 785                 pr_debug("dwc3: invalid parameters\n");
 786                 return -EINVAL;
 787         }
 788
 789         if (!desc->wMaxPacketSize) {
 790                 pr_debug("dwc3: missing wMaxPacketSize\n");
 791                 return -EINVAL;
 792         }
 793
 794         dep = to_dwc3_ep(ep);
 795         dwc = dep->dwc;
 796
 797         if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED,
 798                                         "%s is already enabled\n",
 799                                         dep->name))
 800                 return 0;
 801
 802         spin_lock_irqsave(&dwc->lock, flags);
 803         ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
 804         spin_unlock_irqrestore(&dwc->lock, flags);
 805
 806         return ret;
 807 }
 808
 809 static int dwc3_gadget_ep_disable(struct usb_ep *ep)
 810 {
 811         struct dwc3_ep                  *dep;
 812         struct dwc3                     *dwc;
 813         unsigned long                   flags;
 814         int                             ret;
 815
 816         if (!ep) {
 817                 pr_debug("dwc3: invalid parameters\n");
 818                 return -EINVAL;
 819         }
 820
 821         dep = to_dwc3_ep(ep);
 822         dwc = dep->dwc;
 823
 824         if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED),
 825                                         "%s is already disabled\n",
 826                                         dep->name))
 827                 return 0;
 828
 829         spin_lock_irqsave(&dwc->lock, flags);
 830         ret = __dwc3_gadget_ep_disable(dep);
 831         spin_unlock_irqrestore(&dwc->lock, flags);
 832
 833         return ret;
 834 }
 835
 836 static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep,
 837                 gfp_t gfp_flags)
 838 {
 839         struct dwc3_request             *req;
 840         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
 841
 842         req = kzalloc(sizeof(*req), gfp_flags);
 843         if (!req)
 844                 return NULL;
 845
 846         req->direction  = dep->direction;
 847         req->epnum      = dep->number;
 848         req->dep        = dep;
 849         req->status     = DWC3_REQUEST_STATUS_UNKNOWN;
 850
 851         trace_dwc3_alloc_request(req);
 852
 853         return &req->request;
 854 }
 855
 856 static void dwc3_gadget_ep_free_request(struct usb_ep *ep,
 857                 struct usb_request *request)
 858 {
 859         struct dwc3_request             *req = to_dwc3_request(request);
 860
 861         trace_dwc3_free_request(req);
 862         kfree(req);
 863 }
 864
 865 /**
 866  * dwc3_ep_prev_trb - returns the previous TRB in the ring
 867  * @dep: The endpoint with the TRB ring
 868  * @index: The index of the current TRB in the ring
 869  *
 870  * Returns the TRB prior to the one pointed to by the index. If the
 871  * index is 0, we will wrap backwards, skip the link TRB, and return
 872  * the one just before that.
 873  */
 874 static struct dwc3_trb *dwc3_ep_prev_trb(struct dwc3_ep *dep, u8 index)
 875 {
 876         u8 tmp = index;
 877
 878         if (!tmp)
 879                 tmp = DWC3_TRB_NUM - 1;
 880
 881         return &dep->trb_pool[tmp - 1];
 882 }
 883
 884 static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep)
 885 {
 886         struct dwc3_trb         *tmp;
 887         u8                      trbs_left;
 888
 889         /*
 890          * If enqueue & dequeue are equal than it is either full or empty.
 891          *
 892          * One way to know for sure is if the TRB right before us has HWO bit
 893          * set or not. If it has, then we're definitely full and can't fit any
 894          * more transfers in our ring.
 895          */
 896         if (dep->trb_enqueue == dep->trb_dequeue) {
 897                 tmp = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
 898                 if (tmp->ctrl & DWC3_TRB_CTRL_HWO)
 899                         return 0;
 900
 901                 return DWC3_TRB_NUM - 1;
 902         }
 903
 904         trbs_left = dep->trb_dequeue - dep->trb_enqueue;
 905         trbs_left &= (DWC3_TRB_NUM - 1);
 906
 907         if (dep->trb_dequeue < dep->trb_enqueue)
 908                 trbs_left--;
 909
 910         return trbs_left;
 911 }
 912
 913 static void __dwc3_prepare_one_trb(struct dwc3_ep *dep, struct dwc3_trb *trb,
 914                 dma_addr_t dma, unsigned length, unsigned chain, unsigned node,
 915                 unsigned stream_id, unsigned short_not_ok, unsigned no_interrupt)
 916 {
 917         struct dwc3             *dwc = dep->dwc;
 918         struct usb_gadget       *gadget = &dwc->gadget;
 919         enum usb_device_speed   speed = gadget->speed;
 920
 921         trb->size = DWC3_TRB_SIZE_LENGTH(length);
 922         trb->bpl = lower_32_bits(dma);
 923         trb->bph = upper_32_bits(dma);
 924
 925         switch (usb_endpoint_type(dep->endpoint.desc)) {
 926         case USB_ENDPOINT_XFER_CONTROL:
 927                 trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP;
 928                 break;
 929
 930         case USB_ENDPOINT_XFER_ISOC:
 931                 if (!node) {
 932                         trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;
 933
 934                         /*
 935                          * USB Specification 2.0 Section 5.9.2 states that: "If
 936                          * there is only a single transaction in the microframe,
 937                          * only a DATA0 data packet PID is used.  If there are
 938                          * two transactions per microframe, DATA1 is used for
 939                          * the first transaction data packet and DATA0 is used
 940                          * for the second transaction data packet.  If there are
 941                          * three transactions per microframe, DATA2 is used for
 942                          * the first transaction data packet, DATA1 is used for
 943                          * the second, and DATA0 is used for the third."
 944                          *
 945                          * IOW, we should satisfy the following cases:
 946                          *
 947                          * 1) length <= maxpacket
 948                          *      - DATA0
 949                          *
 950                          * 2) maxpacket < length <= (2 * maxpacket)
 951                          *      - DATA1, DATA0
 952                          *
 953                          * 3) (2 * maxpacket) < length <= (3 * maxpacket)
 954                          *      - DATA2, DATA1, DATA0
 955                          */
 956                         if (speed == USB_SPEED_HIGH) {
 957                                 struct usb_ep *ep = &dep->endpoint;
 958                                 unsigned int mult = 2;
 959                                 unsigned int maxp = usb_endpoint_maxp(ep->desc);
 960
 961                                 if (length <= (2 * maxp))
 962                                         mult--;
 963
 964                                 if (length <= maxp)
 965                                         mult--;
 966
 967                                 trb->size |= DWC3_TRB_SIZE_PCM1(mult);
 968                         }
 969                 } else {
 970                         trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS;
 971                 }
 972
 973                 /* always enable Interrupt on Missed ISOC */
 974                 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
 975                 break;
 976
 977         case USB_ENDPOINT_XFER_BULK:
 978         case USB_ENDPOINT_XFER_INT:
 979                 trb->ctrl = DWC3_TRBCTL_NORMAL;
 980                 break;
 981         default:
 982                 /*
 983                  * This is only possible with faulty memory because we
 984                  * checked it already :)
 985                  */
 986                 dev_WARN(dwc->dev, "Unknown endpoint type %d\n",
 987                                 usb_endpoint_type(dep->endpoint.desc));
 988         }
 989
 990         /*
 991          * Enable Continue on Short Packet
 992          * when endpoint is not a stream capable
 993          */
 994         if (usb_endpoint_dir_out(dep->endpoint.desc)) {
 995                 if (!dep->stream_capable)
 996                         trb->ctrl |= DWC3_TRB_CTRL_CSP;
 997
 998                 if (short_not_ok)
 999                         trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
1000         }
1001
1002         if ((!no_interrupt && !chain) ||
1003                         (dwc3_calc_trbs_left(dep) == 1))
1004                 trb->ctrl |= DWC3_TRB_CTRL_IOC;
1005
1006         if (chain)
1007                 trb->ctrl |= DWC3_TRB_CTRL_CHN;
1008
1009         if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable)
1010                 trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(stream_id);
1011
1012         trb->ctrl |= DWC3_TRB_CTRL_HWO;
1013
1014         dwc3_ep_inc_enq(dep);
1015
1016         trace_dwc3_prepare_trb(dep, trb);
1017 }
1018
1019 /**
1020  * dwc3_prepare_one_trb - setup one TRB from one request
1021  * @dep: endpoint for which this request is prepared
1022  * @req: dwc3_request pointer
1023  * @chain: should this TRB be chained to the next?
1024  * @node: only for isochronous endpoints. First TRB needs different type.
1025  */
1026 static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
1027                 struct dwc3_request *req, unsigned chain, unsigned node)
1028 {
1029         struct dwc3_trb         *trb;
1030         unsigned int            length;
1031         dma_addr_t              dma;
1032         unsigned                stream_id = req->request.stream_id;
1033         unsigned                short_not_ok = req->request.short_not_ok;
1034         unsigned                no_interrupt = req->request.no_interrupt;
1035
1036         if (req->request.num_sgs > 0) {
1037                 length = sg_dma_len(req->start_sg);
1038                 dma = sg_dma_address(req->start_sg);
1039         } else {
1040                 length = req->request.length;
1041                 dma = req->request.dma;
1042         }
1043
1044         trb = &dep->trb_pool[dep->trb_enqueue];
1045
1046         if (!req->trb) {
1047                 dwc3_gadget_move_started_request(req);
1048                 req->trb = trb;
1049                 req->trb_dma = dwc3_trb_dma_offset(dep, trb);
1050         }
1051
1052         req->num_trbs++;
1053
1054         __dwc3_prepare_one_trb(dep, trb, dma, length, chain, node,
1055                         stream_id, short_not_ok, no_interrupt);
1056 }
1057
1058 static void dwc3_prepare_one_trb_sg(struct dwc3_ep *dep,
1059                 struct dwc3_request *req)
1060 {
1061         struct scatterlist *sg = req->start_sg;
1062         struct scatterlist *s;
1063         int             i;
1064
1065         unsigned int remaining = req->request.num_mapped_sgs
1066                 - req->num_queued_sgs;
1067
1068         for_each_sg(sg, s, remaining, i) {
1069                 unsigned int length = req->request.length;
1070                 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1071                 unsigned int rem = length % maxp;
1072                 unsigned chain = true;
1073
1074                 /*
1075                  * IOMMU driver is coalescing the list of sgs which shares a
1076                  * page boundary into one and giving it to USB driver. With
1077                  * this the number of sgs mapped is not equal to the number of
1078                  * sgs passed. So mark the chain bit to false if it isthe last
1079                  * mapped sg.
1080                  */
1081                 if (i == remaining - 1)
1082                         chain = false;
1083
1084                 if (rem && usb_endpoint_dir_out(dep->endpoint.desc) && !chain) {
1085                         struct dwc3     *dwc = dep->dwc;
1086                         struct dwc3_trb *trb;
1087
1088                         req->needs_extra_trb = true;
1089
1090                         /* prepare normal TRB */
1091                         dwc3_prepare_one_trb(dep, req, true, i);
1092
1093                         /* Now prepare one extra TRB to align transfer size */
1094                         trb = &dep->trb_pool[dep->trb_enqueue];
1095                         req->num_trbs++;
1096                         __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr,
1097                                         maxp - rem, false, 1,
1098                                         req->request.stream_id,
1099                                         req->request.short_not_ok,
1100                                         req->request.no_interrupt);
1101                 } else {
1102                         dwc3_prepare_one_trb(dep, req, chain, i);
1103                 }
1104
1105                 /*
1106                  * There can be a situation where all sgs in sglist are not
1107                  * queued because of insufficient trb number. To handle this
1108                  * case, update start_sg to next sg to be queued, so that
1109                  * we have free trbs we can continue queuing from where we
1110                  * previously stopped
1111                  */
1112                 if (chain)
1113                         req->start_sg = sg_next(s);
1114
1115                 req->num_queued_sgs++;
1116
1117                 if (!dwc3_calc_trbs_left(dep))
1118                         break;
1119         }
1120 }
1121
1122 static void dwc3_prepare_one_trb_linear(struct dwc3_ep *dep,
1123                 struct dwc3_request *req)
1124 {
1125         unsigned int length = req->request.length;
1126         unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1127         unsigned int rem = length % maxp;
1128
1129         if ((!length || rem) && usb_endpoint_dir_out(dep->endpoint.desc)) {
1130                 struct dwc3     *dwc = dep->dwc;
1131                 struct dwc3_trb *trb;
1132
1133                 req->needs_extra_trb = true;
1134
1135                 /* prepare normal TRB */
1136                 dwc3_prepare_one_trb(dep, req, true, 0);
1137
1138                 /* Now prepare one extra TRB to align transfer size */
1139                 trb = &dep->trb_pool[dep->trb_enqueue];
1140                 req->num_trbs++;
1141                 __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, maxp - rem,
1142                                 false, 1, req->request.stream_id,
1143                                 req->request.short_not_ok,
1144                                 req->request.no_interrupt);
1145         } else if (req->request.zero && req->request.length &&
1146                    (IS_ALIGNED(req->request.length, maxp))) {
1147                 struct dwc3     *dwc = dep->dwc;
1148                 struct dwc3_trb *trb;
1149
1150                 req->needs_extra_trb = true;
1151
1152                 /* prepare normal TRB */
1153                 dwc3_prepare_one_trb(dep, req, true, 0);
1154
1155                 /* Now prepare one extra TRB to handle ZLP */
1156                 trb = &dep->trb_pool[dep->trb_enqueue];
1157                 req->num_trbs++;
1158                 __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, 0,
1159                                 false, 1, req->request.stream_id,
1160                                 req->request.short_not_ok,
1161                                 req->request.no_interrupt);
1162         } else {
1163                 dwc3_prepare_one_trb(dep, req, false, 0);
1164         }
1165 }
1166
1167 /*
1168  * dwc3_prepare_trbs - setup TRBs from requests
1169  * @dep: endpoint for which requests are being prepared
1170  *
1171  * The function goes through the requests list and sets up TRBs for the
1172  * transfers. The function returns once there are no more TRBs available or
1173  * it runs out of requests.
1174  */
1175 static void dwc3_prepare_trbs(struct dwc3_ep *dep)
1176 {
1177         struct dwc3_request     *req, *n;
1178
1179         BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM);
1180
1181         /*
1182          * We can get in a situation where there's a request in the started list
1183          * but there weren't enough TRBs to fully kick it in the first time
1184          * around, so it has been waiting for more TRBs to be freed up.
1185          *
1186          * In that case, we should check if we have a request with pending_sgs
1187          * in the started list and prepare TRBs for that request first,
1188          * otherwise we will prepare TRBs completely out of order and that will
1189          * break things.
1190          */
1191         list_for_each_entry(req, &dep->started_list, list) {
1192                 if (req->num_pending_sgs > 0)
1193                         dwc3_prepare_one_trb_sg(dep, req);
1194
1195                 if (!dwc3_calc_trbs_left(dep))
1196                         return;
1197         }
1198
1199         list_for_each_entry_safe(req, n, &dep->pending_list, list) {
1200                 struct dwc3     *dwc = dep->dwc;
1201                 int             ret;
1202
1203                 ret = usb_gadget_map_request_by_dev(dwc->sysdev, &req->request,
1204                                                     dep->direction);
1205                 if (ret)
1206                         return;
1207
1208                 req->sg                 = req->request.sg;
1209                 req->start_sg           = req->sg;
1210                 req->num_queued_sgs     = 0;
1211                 req->num_pending_sgs    = req->request.num_mapped_sgs;
1212
1213                 if (req->num_pending_sgs > 0)
1214                         dwc3_prepare_one_trb_sg(dep, req);
1215                 else
1216                         dwc3_prepare_one_trb_linear(dep, req);
1217
1218                 if (!dwc3_calc_trbs_left(dep))
1219                         return;
1220         }
1221 }
1222
1223 static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep)
1224 {
1225         struct dwc3_gadget_ep_cmd_params params;
1226         struct dwc3_request             *req;
1227         int                             starting;
1228         int                             ret;
1229         u32                             cmd;
1230
1231         if (!dwc3_calc_trbs_left(dep))
1232                 return 0;
1233
1234         starting = !(dep->flags & DWC3_EP_TRANSFER_STARTED);
1235
1236         dwc3_prepare_trbs(dep);
1237         req = next_request(&dep->started_list);
1238         if (!req) {
1239                 dep->flags |= DWC3_EP_PENDING_REQUEST;
1240                 return 0;
1241         }
1242
1243         memset(&params, 0, sizeof(params));
1244
1245         if (starting) {
1246                 params.param0 = upper_32_bits(req->trb_dma);
1247                 params.param1 = lower_32_bits(req->trb_dma);
1248                 cmd = DWC3_DEPCMD_STARTTRANSFER;
1249
1250                 if (dep->stream_capable)
1251                         cmd |= DWC3_DEPCMD_PARAM(req->request.stream_id);
1252
1253                 if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
1254                         cmd |= DWC3_DEPCMD_PARAM(dep->frame_number);
1255         } else {
1256                 cmd = DWC3_DEPCMD_UPDATETRANSFER |
1257                         DWC3_DEPCMD_PARAM(dep->resource_index);
1258         }
1259
1260         ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
1261         if (ret < 0) {
1262                 /*
1263                  * FIXME we need to iterate over the list of requests
1264                  * here and stop, unmap, free and del each of the linked
1265                  * requests instead of what we do now.
1266                  */
1267                 if (req->trb)
1268                         memset(req->trb, 0, sizeof(struct dwc3_trb));
1269                 dwc3_gadget_del_and_unmap_request(dep, req, ret);
1270                 return ret;
1271         }
1272
1273         return 0;
1274 }
1275
1276 static int __dwc3_gadget_get_frame(struct dwc3 *dwc)
1277 {
1278         u32                     reg;
1279
1280         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1281         return DWC3_DSTS_SOFFN(reg);
1282 }
1283
1284 /**
1285  * dwc3_gadget_start_isoc_quirk - workaround invalid frame number
1286  * @dep: isoc endpoint
1287  *
1288  * This function tests for the correct combination of BIT[15:14] from the 16-bit
1289  * microframe number reported by the XferNotReady event for the future frame
1290  * number to start the isoc transfer.
1291  *
1292  * In DWC_usb31 version 1.70a-ea06 and prior, for highspeed and fullspeed
1293  * isochronous IN, BIT[15:14] of the 16-bit microframe number reported by the
1294  * XferNotReady event are invalid. The driver uses this number to schedule the
1295  * isochronous transfer and passes it to the START TRANSFER command. Because
1296  * this number is invalid, the command may fail. If BIT[15:14] matches the
1297  * internal 16-bit microframe, the START TRANSFER command will pass and the
1298  * transfer will start at the scheduled time, if it is off by 1, the command
1299  * will still pass, but the transfer will start 2 seconds in the future. For all
1300  * other conditions, the START TRANSFER command will fail with bus-expiry.
1301  *
1302  * In order to workaround this issue, we can test for the correct combination of
1303  * BIT[15:14] by sending START TRANSFER commands with different values of
1304  * BIT[15:14]: 'b00, 'b01, 'b10, and 'b11. Each combination is 2^14 uframe apart
1305  * (or 2 seconds). 4 seconds into the future will result in a bus-expiry status.
1306  * As the result, within the 4 possible combinations for BIT[15:14], there will
1307  * be 2 successful and 2 failure START COMMAND status. One of the 2 successful
1308  * command status will result in a 2-second delay start. The smaller BIT[15:14]
1309  * value is the correct combination.
1310  *
1311  * Since there are only 4 outcomes and the results are ordered, we can simply
1312  * test 2 START TRANSFER commands with BIT[15:14] combinations 'b00 and 'b01 to
1313  * deduce the smaller successful combination.
1314  *
1315  * Let test0 = test status for combination 'b00 and test1 = test status for 'b01
1316  * of BIT[15:14]. The correct combination is as follow:
1317  *
1318  * if test0 fails and test1 passes, BIT[15:14] is 'b01
1319  * if test0 fails and test1 fails, BIT[15:14] is 'b10
1320  * if test0 passes and test1 fails, BIT[15:14] is 'b11
1321  * if test0 passes and test1 passes, BIT[15:14] is 'b00
1322  *
1323  * Synopsys STAR 9001202023: Wrong microframe number for isochronous IN
1324  * endpoints.
1325  */
1326 static int dwc3_gadget_start_isoc_quirk(struct dwc3_ep *dep)
1327 {
1328         int cmd_status = 0;
1329         bool test0;
1330         bool test1;
1331
1332         while (dep->combo_num < 2) {
1333                 struct dwc3_gadget_ep_cmd_params params;
1334                 u32 test_frame_number;
1335                 u32 cmd;
1336
1337                 /*
1338                  * Check if we can start isoc transfer on the next interval or
1339                  * 4 uframes in the future with BIT[15:14] as dep->combo_num
1340                  */
1341                 test_frame_number = dep->frame_number & 0x3fff;
1342                 test_frame_number |= dep->combo_num << 14;
1343                 test_frame_number += max_t(u32, 4, dep->interval);
1344
1345                 params.param0 = upper_32_bits(dep->dwc->bounce_addr);
1346                 params.param1 = lower_32_bits(dep->dwc->bounce_addr);
1347
1348                 cmd = DWC3_DEPCMD_STARTTRANSFER;
1349                 cmd |= DWC3_DEPCMD_PARAM(test_frame_number);
1350                 cmd_status = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
1351
1352                 /* Redo if some other failure beside bus-expiry is received */
1353                 if (cmd_status && cmd_status != -EAGAIN) {
1354                         dep->start_cmd_status = 0;
1355                         dep->combo_num = 0;
1356                         return 0;
1357                 }
1358
1359                 /* Store the first test status */
1360                 if (dep->combo_num == 0)
1361                         dep->start_cmd_status = cmd_status;
1362
1363                 dep->combo_num++;
1364
1365                 /*
1366                  * End the transfer if the START_TRANSFER command is successful
1367                  * to wait for the next XferNotReady to test the command again
1368                  */
1369                 if (cmd_status == 0) {
1370                         dwc3_stop_active_transfer(dep, true, true);
1371                         return 0;
1372                 }
1373         }
1374
1375         /* test0 and test1 are both completed at this point */
1376         test0 = (dep->start_cmd_status == 0);
1377         test1 = (cmd_status == 0);
1378
1379         if (!test0 && test1)
1380                 dep->combo_num = 1;
1381         else if (!test0 && !test1)
1382                 dep->combo_num = 2;
1383         else if (test0 && !test1)
1384                 dep->combo_num = 3;
1385         else if (test0 && test1)
1386                 dep->combo_num = 0;
1387
1388         dep->frame_number &= 0x3fff;
1389         dep->frame_number |= dep->combo_num << 14;
1390         dep->frame_number += max_t(u32, 4, dep->interval);
1391
1392         /* Reinitialize test variables */
1393         dep->start_cmd_status = 0;
1394         dep->combo_num = 0;
1395
1396         return __dwc3_gadget_kick_transfer(dep);
1397 }
1398
1399 static int __dwc3_gadget_start_isoc(struct dwc3_ep *dep)
1400 {
1401         struct dwc3 *dwc = dep->dwc;
1402         int ret;
1403         int i;
1404
1405         if (list_empty(&dep->pending_list)) {
1406                 dep->flags |= DWC3_EP_PENDING_REQUEST;
1407                 return -EAGAIN;
1408         }
1409
1410         if (!dwc->dis_start_transfer_quirk && dwc3_is_usb31(dwc) &&
1411             (dwc->revision <= DWC3_USB31_REVISION_160A ||
1412              (dwc->revision == DWC3_USB31_REVISION_170A &&
1413               dwc->version_type >= DWC31_VERSIONTYPE_EA01 &&
1414               dwc->version_type <= DWC31_VERSIONTYPE_EA06))) {
1415
1416                 if (dwc->gadget.speed <= USB_SPEED_HIGH && dep->direction)
1417                         return dwc3_gadget_start_isoc_quirk(dep);
1418         }
1419
1420         for (i = 0; i < DWC3_ISOC_MAX_RETRIES; i++) {
1421                 dep->frame_number = DWC3_ALIGN_FRAME(dep, i + 1);
1422
1423                 ret = __dwc3_gadget_kick_transfer(dep);
1424                 if (ret != -EAGAIN)
1425                         break;
1426         }
1427
1428         return ret;
1429 }
1430
1431 static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
1432 {
1433         struct dwc3             *dwc = dep->dwc;
1434
1435         if (!dep->endpoint.desc) {
1436                 dev_err(dwc->dev, "%s: can't queue to disabled endpoint\n",
1437                                 dep->name);
1438                 return -ESHUTDOWN;
1439         }
1440
1441         if (WARN(req->dep != dep, "request %pK belongs to '%s'\n",
1442                                 &req->request, req->dep->name))
1443                 return -EINVAL;
1444
1445         if (WARN(req->status < DWC3_REQUEST_STATUS_COMPLETED,
1446                                 "%s: request %pK already in flight\n",
1447                                 dep->name, &req->request))
1448                 return -EINVAL;
1449
1450         pm_runtime_get(dwc->dev);
1451
1452         req->request.actual     = 0;
1453         req->request.status     = -EINPROGRESS;
1454
1455         trace_dwc3_ep_queue(req);
1456
1457         list_add_tail(&req->list, &dep->pending_list);
1458         req->status = DWC3_REQUEST_STATUS_QUEUED;
1459
1460         /* Start the transfer only after the END_TRANSFER is completed */
1461         if (dep->flags & DWC3_EP_END_TRANSFER_PENDING) {
1462                 dep->flags |= DWC3_EP_DELAY_START;
1463                 return 0;
1464         }
1465
1466         /*
1467          * NOTICE: Isochronous endpoints should NEVER be prestarted. We must
1468          * wait for a XferNotReady event so we will know what's the current
1469          * (micro-)frame number.
1470          *
1471          * Without this trick, we are very, very likely gonna get Bus Expiry
1472          * errors which will force us issue EndTransfer command.
1473          */
1474         if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1475                 if (!(dep->flags & DWC3_EP_PENDING_REQUEST) &&
1476                                 !(dep->flags & DWC3_EP_TRANSFER_STARTED))
1477                         return 0;
1478
1479                 if ((dep->flags & DWC3_EP_PENDING_REQUEST)) {
1480                         if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) {
1481                                 return __dwc3_gadget_start_isoc(dep);
1482                         }
1483                 }
1484         }
1485
1486         return __dwc3_gadget_kick_transfer(dep);
1487 }
1488
1489 static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
1490         gfp_t gfp_flags)
1491 {
1492         struct dwc3_request             *req = to_dwc3_request(request);
1493         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1494         struct dwc3                     *dwc = dep->dwc;
1495
1496         unsigned long                   flags;
1497
1498         int                             ret;
1499
1500         spin_lock_irqsave(&dwc->lock, flags);
1501         ret = __dwc3_gadget_ep_queue(dep, req);
1502         spin_unlock_irqrestore(&dwc->lock, flags);
1503
1504         return ret;
1505 }
1506
1507 static void dwc3_gadget_ep_skip_trbs(struct dwc3_ep *dep, struct dwc3_request *req)
1508 {
1509         int i;
1510
1511         /* If req->trb is not set, then the request has not started */
1512         if (!req->trb)
1513                 return;
1514
1515         /*
1516          * If request was already started, this means we had to
1517          * stop the transfer. With that we also need to ignore
1518          * all TRBs used by the request, however TRBs can only
1519          * be modified after completion of END_TRANSFER
1520          * command. So what we do here is that we wait for
1521          * END_TRANSFER completion and only after that, we jump
1522          * over TRBs by clearing HWO and incrementing dequeue
1523          * pointer.
1524          */
1525         for (i = 0; i < req->num_trbs; i++) {
1526                 struct dwc3_trb *trb;
1527
1528                 trb = &dep->trb_pool[dep->trb_dequeue];
1529                 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
1530                 dwc3_ep_inc_deq(dep);
1531         }
1532
1533         req->num_trbs = 0;
1534 }
1535
1536 static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep)
1537 {
1538         struct dwc3_request             *req;
1539         struct dwc3_request             *tmp;
1540
1541         list_for_each_entry_safe(req, tmp, &dep->cancelled_list, list) {
1542                 dwc3_gadget_ep_skip_trbs(dep, req);
1543                 dwc3_gadget_giveback(dep, req, -ECONNRESET);
1544         }
1545 }
1546
1547 static int dwc3_gadget_ep_dequeue(struct usb_ep *ep,
1548                 struct usb_request *request)
1549 {
1550         struct dwc3_request             *req = to_dwc3_request(request);
1551         struct dwc3_request             *r = NULL;
1552
1553         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1554         struct dwc3                     *dwc = dep->dwc;
1555
1556         unsigned long                   flags;
1557         int                             ret = 0;
1558
1559         trace_dwc3_ep_dequeue(req);
1560
1561         spin_lock_irqsave(&dwc->lock, flags);
1562
1563         list_for_each_entry(r, &dep->pending_list, list) {
1564                 if (r == req)
1565                         break;
1566         }
1567
1568         if (r != req) {
1569                 list_for_each_entry(r, &dep->started_list, list) {
1570                         if (r == req)
1571                                 break;
1572                 }
1573                 if (r == req) {
1574                         /* wait until it is processed */
1575                         dwc3_stop_active_transfer(dep, true, true);
1576
1577                         if (!r->trb)
1578                                 goto out0;
1579
1580                         dwc3_gadget_move_cancelled_request(req);
1581                         if (dep->flags & DWC3_EP_TRANSFER_STARTED)
1582                                 goto out0;
1583                         else
1584                                 goto out1;
1585                 }
1586                 dev_err(dwc->dev, "request %pK was not queued to %s\n",
1587                                 request, ep->name);
1588                 ret = -EINVAL;
1589                 goto out0;
1590         }
1591
1592 out1:
1593         dwc3_gadget_giveback(dep, req, -ECONNRESET);
1594
1595 out0:
1596         spin_unlock_irqrestore(&dwc->lock, flags);
1597
1598         return ret;
1599 }
1600
1601 int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol)
1602 {
1603         struct dwc3_gadget_ep_cmd_params        params;
1604         struct dwc3                             *dwc = dep->dwc;
1605         struct dwc3_request                     *req;
1606         struct dwc3_request                     *tmp;
1607         int                                     ret;
1608
1609         if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1610                 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
1611                 return -EINVAL;
1612         }
1613
1614         memset(&params, 0x00, sizeof(params));
1615
1616         if (value) {
1617                 struct dwc3_trb *trb;
1618
1619                 unsigned transfer_in_flight;
1620                 unsigned started;
1621
1622                 if (dep->number > 1)
1623                         trb = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
1624                 else
1625                         trb = &dwc->ep0_trb[dep->trb_enqueue];
1626
1627                 transfer_in_flight = trb->ctrl & DWC3_TRB_CTRL_HWO;
1628                 started = !list_empty(&dep->started_list);
1629
1630                 if (!protocol && ((dep->direction && transfer_in_flight) ||
1631                                 (!dep->direction && started))) {
1632                         return -EAGAIN;
1633                 }
1634
1635                 ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETSTALL,
1636                                 &params);
1637                 if (ret)
1638                         dev_err(dwc->dev, "failed to set STALL on %s\n",
1639                                         dep->name);
1640                 else
1641                         dep->flags |= DWC3_EP_STALL;
1642         } else {
1643                 /*
1644                  * Don't issue CLEAR_STALL command to control endpoints. The
1645                  * controller automatically clears the STALL when it receives
1646                  * the SETUP token.
1647                  */
1648                 if (dep->number <= 1) {
1649                         dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
1650                         return 0;
1651                 }
1652
1653                 ret = dwc3_send_clear_stall_ep_cmd(dep);
1654                 if (ret) {
1655                         dev_err(dwc->dev, "failed to clear STALL on %s\n",
1656                                         dep->name);
1657                         return ret;
1658                 }
1659
1660                 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
1661
1662                 dwc3_stop_active_transfer(dep, true, true);
1663
1664                 list_for_each_entry_safe(req, tmp, &dep->started_list, list)
1665                         dwc3_gadget_move_cancelled_request(req);
1666
1667                 list_for_each_entry_safe(req, tmp, &dep->pending_list, list)
1668                         dwc3_gadget_move_cancelled_request(req);
1669
1670                 if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING)) {
1671                         dep->flags &= ~DWC3_EP_DELAY_START;
1672                         dwc3_gadget_ep_cleanup_cancelled_requests(dep);
1673                 }
1674         }
1675
1676         return ret;
1677 }
1678
1679 static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value)
1680 {
1681         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1682         struct dwc3                     *dwc = dep->dwc;
1683
1684         unsigned long                   flags;
1685
1686         int                             ret;
1687
1688         spin_lock_irqsave(&dwc->lock, flags);
1689         ret = __dwc3_gadget_ep_set_halt(dep, value, false);
1690         spin_unlock_irqrestore(&dwc->lock, flags);
1691
1692         return ret;
1693 }
1694
1695 static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep)
1696 {
1697         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1698         struct dwc3                     *dwc = dep->dwc;
1699         unsigned long                   flags;
1700         int                             ret;
1701
1702         spin_lock_irqsave(&dwc->lock, flags);
1703         dep->flags |= DWC3_EP_WEDGE;
1704
1705         if (dep->number == 0 || dep->number == 1)
1706                 ret = __dwc3_gadget_ep0_set_halt(ep, 1);
1707         else
1708                 ret = __dwc3_gadget_ep_set_halt(dep, 1, false);
1709         spin_unlock_irqrestore(&dwc->lock, flags);
1710
1711         return ret;
1712 }
1713
1714 /* -------------------------------------------------------------------------- */
1715
1716 static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = {
1717         .bLength        = USB_DT_ENDPOINT_SIZE,
1718         .bDescriptorType = USB_DT_ENDPOINT,
1719         .bmAttributes   = USB_ENDPOINT_XFER_CONTROL,
1720 };
1721
1722 static const struct usb_ep_ops dwc3_gadget_ep0_ops = {
1723         .enable         = dwc3_gadget_ep0_enable,
1724         .disable        = dwc3_gadget_ep0_disable,
1725         .alloc_request  = dwc3_gadget_ep_alloc_request,
1726         .free_request   = dwc3_gadget_ep_free_request,
1727         .queue          = dwc3_gadget_ep0_queue,
1728         .dequeue        = dwc3_gadget_ep_dequeue,
1729         .set_halt       = dwc3_gadget_ep0_set_halt,
1730         .set_wedge      = dwc3_gadget_ep_set_wedge,
1731 };
1732
1733 static const struct usb_ep_ops dwc3_gadget_ep_ops = {
1734         .enable         = dwc3_gadget_ep_enable,
1735         .disable        = dwc3_gadget_ep_disable,
1736         .alloc_request  = dwc3_gadget_ep_alloc_request,
1737         .free_request   = dwc3_gadget_ep_free_request,
1738         .queue          = dwc3_gadget_ep_queue,
1739         .dequeue        = dwc3_gadget_ep_dequeue,
1740         .set_halt       = dwc3_gadget_ep_set_halt,
1741         .set_wedge      = dwc3_gadget_ep_set_wedge,
1742 };
1743
1744 /* -------------------------------------------------------------------------- */
1745
1746 static int dwc3_gadget_get_frame(struct usb_gadget *g)
1747 {
1748         struct dwc3             *dwc = gadget_to_dwc(g);
1749
1750         return __dwc3_gadget_get_frame(dwc);
1751 }
1752
1753 static int __dwc3_gadget_wakeup(struct dwc3 *dwc)
1754 {
1755         int                     retries;
1756
1757         int                     ret;
1758         u32                     reg;
1759
1760         u8                      link_state;
1761
1762         /*
1763          * According to the Databook Remote wakeup request should
1764          * be issued only when the device is in early suspend state.
1765          *
1766          * We can check that via USB Link State bits in DSTS register.
1767          */
1768         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1769
1770         link_state = DWC3_DSTS_USBLNKST(reg);
1771
1772         switch (link_state) {
1773         case DWC3_LINK_STATE_RESET:
1774         case DWC3_LINK_STATE_RX_DET:    /* in HS, means Early Suspend */
1775         case DWC3_LINK_STATE_U3:        /* in HS, means SUSPEND */
1776         case DWC3_LINK_STATE_RESUME:
1777                 break;
1778         default:
1779                 return -EINVAL;
1780         }
1781
1782         ret = dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RECOV);
1783         if (ret < 0) {
1784                 dev_err(dwc->dev, "failed to put link in Recovery\n");
1785                 return ret;
1786         }
1787
1788         /* Recent versions do this automatically */
1789         if (dwc->revision < DWC3_REVISION_194A) {
1790                 /* write zeroes to Link Change Request */
1791                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1792                 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
1793                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1794         }
1795
1796         /* poll until Link State changes to ON */
1797         retries = 20000;
1798
1799         while (retries--) {
1800                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1801
1802                 /* in HS, means ON */
1803                 if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0)
1804                         break;
1805         }
1806
1807         if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) {
1808                 dev_err(dwc->dev, "failed to send remote wakeup\n");
1809                 return -EINVAL;
1810         }
1811
1812         return 0;
1813 }
1814
1815 static int dwc3_gadget_wakeup(struct usb_gadget *g)
1816 {
1817         struct dwc3             *dwc = gadget_to_dwc(g);
1818         unsigned long           flags;
1819         int                     ret;
1820
1821         spin_lock_irqsave(&dwc->lock, flags);
1822         ret = __dwc3_gadget_wakeup(dwc);
1823         spin_unlock_irqrestore(&dwc->lock, flags);
1824
1825         return ret;
1826 }
1827
1828 static int dwc3_gadget_set_selfpowered(struct usb_gadget *g,
1829                 int is_selfpowered)
1830 {
1831         struct dwc3             *dwc = gadget_to_dwc(g);
1832         unsigned long           flags;
1833
1834         spin_lock_irqsave(&dwc->lock, flags);
1835         g->is_selfpowered = !!is_selfpowered;
1836         spin_unlock_irqrestore(&dwc->lock, flags);
1837
1838         return 0;
1839 }
1840
1841 static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on, int suspend)
1842 {
1843         u32                     reg;
1844         u32                     timeout = 500;
1845
1846         if (pm_runtime_suspended(dwc->dev))
1847                 return 0;
1848
1849         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1850         if (is_on) {
1851                 if (dwc->revision <= DWC3_REVISION_187A) {
1852                         reg &= ~DWC3_DCTL_TRGTULST_MASK;
1853                         reg |= DWC3_DCTL_TRGTULST_RX_DET;
1854                 }
1855
1856                 if (dwc->revision >= DWC3_REVISION_194A)
1857                         reg &= ~DWC3_DCTL_KEEP_CONNECT;
1858                 reg |= DWC3_DCTL_RUN_STOP;
1859
1860                 if (dwc->has_hibernation)
1861                         reg |= DWC3_DCTL_KEEP_CONNECT;
1862
1863                 dwc->pullups_connected = true;
1864         } else {
1865                 reg &= ~DWC3_DCTL_RUN_STOP;
1866
1867                 if (dwc->has_hibernation && !suspend)
1868                         reg &= ~DWC3_DCTL_KEEP_CONNECT;
1869
1870                 dwc->pullups_connected = false;
1871         }
1872
1873         dwc3_gadget_dctl_write_safe(dwc, reg);
1874
1875         do {
1876                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1877                 reg &= DWC3_DSTS_DEVCTRLHLT;
1878         } while (--timeout && !(!is_on ^ !reg));
1879
1880         if (!timeout)
1881                 return -ETIMEDOUT;
1882
1883         return 0;
1884 }
1885
1886 static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
1887 {
1888         struct dwc3             *dwc = gadget_to_dwc(g);
1889         unsigned long           flags;
1890         int                     ret;
1891
1892         is_on = !!is_on;
1893
1894         /*
1895          * Per databook, when we want to stop the gadget, if a control transfer
1896          * is still in process, complete it and get the core into setup phase.
1897          */
1898         if (!is_on && dwc->ep0state != EP0_SETUP_PHASE) {
1899                 reinit_completion(&dwc->ep0_in_setup);
1900
1901                 ret = wait_for_completion_timeout(&dwc->ep0_in_setup,
1902                                 msecs_to_jiffies(DWC3_PULL_UP_TIMEOUT));
1903                 if (ret == 0) {
1904                         dev_err(dwc->dev, "timed out waiting for SETUP phase\n");
1905                         return -ETIMEDOUT;
1906                 }
1907         }
1908
1909         spin_lock_irqsave(&dwc->lock, flags);
1910         ret = dwc3_gadget_run_stop(dwc, is_on, false);
1911         spin_unlock_irqrestore(&dwc->lock, flags);
1912
1913         return ret;
1914 }
1915
1916 static void dwc3_gadget_enable_irq(struct dwc3 *dwc)
1917 {
1918         u32                     reg;
1919
1920         /* Enable all but Start and End of Frame IRQs */
1921         reg = (DWC3_DEVTEN_VNDRDEVTSTRCVEDEN |
1922                         DWC3_DEVTEN_EVNTOVERFLOWEN |
1923                         DWC3_DEVTEN_CMDCMPLTEN |
1924                         DWC3_DEVTEN_ERRTICERREN |
1925                         DWC3_DEVTEN_WKUPEVTEN |
1926                         DWC3_DEVTEN_CONNECTDONEEN |
1927                         DWC3_DEVTEN_USBRSTEN |
1928                         DWC3_DEVTEN_DISCONNEVTEN);
1929
1930         if (dwc->revision < DWC3_REVISION_250A)
1931                 reg |= DWC3_DEVTEN_ULSTCNGEN;
1932
1933         dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
1934 }
1935
1936 static void dwc3_gadget_disable_irq(struct dwc3 *dwc)
1937 {
1938         /* mask all interrupts */
1939         dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);
1940 }
1941
1942 static irqreturn_t dwc3_interrupt(int irq, void *_dwc);
1943 static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc);
1944
1945 /**
1946  * dwc3_gadget_setup_nump - calculate and initialize NUMP field of %DWC3_DCFG
1947  * @dwc: pointer to our context structure
1948  *
1949  * The following looks like complex but it's actually very simple. In order to
1950  * calculate the number of packets we can burst at once on OUT transfers, we're
1951  * gonna use RxFIFO size.
1952  *
1953  * To calculate RxFIFO size we need two numbers:
1954  * MDWIDTH = size, in bits, of the internal memory bus
1955  * RAM2_DEPTH = depth, in MDWIDTH, of internal RAM2 (where RxFIFO sits)
1956  *
1957  * Given these two numbers, the formula is simple:
1958  *
1959  * RxFIFO Size = (RAM2_DEPTH * MDWIDTH / 8) - 24 - 16;
1960  *
1961  * 24 bytes is for 3x SETUP packets
1962  * 16 bytes is a clock domain crossing tolerance
1963  *
1964  * Given RxFIFO Size, NUMP = RxFIFOSize / 1024;
1965  */
1966 static void dwc3_gadget_setup_nump(struct dwc3 *dwc)
1967 {
1968         u32 ram2_depth;
1969         u32 mdwidth;
1970         u32 nump;
1971         u32 reg;
1972
1973         ram2_depth = DWC3_GHWPARAMS7_RAM2_DEPTH(dwc->hwparams.hwparams7);
1974         mdwidth = DWC3_GHWPARAMS0_MDWIDTH(dwc->hwparams.hwparams0);
1975
1976         nump = ((ram2_depth * mdwidth / 8) - 24 - 16) / 1024;
1977         nump = min_t(u32, nump, 16);
1978
1979         /* update NumP */
1980         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
1981         reg &= ~DWC3_DCFG_NUMP_MASK;
1982         reg |= nump << DWC3_DCFG_NUMP_SHIFT;
1983         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
1984 }
1985
1986 static int __dwc3_gadget_start(struct dwc3 *dwc)
1987 {
1988         struct dwc3_ep          *dep;
1989         int                     ret = 0;
1990         u32                     reg;
1991
1992         /*
1993          * Use IMOD if enabled via dwc->imod_interval. Otherwise, if
1994          * the core supports IMOD, disable it.
1995          */
1996         if (dwc->imod_interval) {
1997                 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
1998                 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
1999         } else if (dwc3_has_imod(dwc)) {
2000                 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), 0);
2001         }
2002
2003         /*
2004          * We are telling dwc3 that we want to use DCFG.NUMP as ACK TP's NUMP
2005          * field instead of letting dwc3 itself calculate that automatically.
2006          *
2007          * This way, we maximize the chances that we'll be able to get several
2008          * bursts of data without going through any sort of endpoint throttling.
2009          */
2010         reg = dwc3_readl(dwc->regs, DWC3_GRXTHRCFG);
2011         if (dwc3_is_usb31(dwc))
2012                 reg &= ~DWC31_GRXTHRCFG_PKTCNTSEL;
2013         else
2014                 reg &= ~DWC3_GRXTHRCFG_PKTCNTSEL;
2015
2016         dwc3_writel(dwc->regs, DWC3_GRXTHRCFG, reg);
2017
2018         dwc3_gadget_setup_nump(dwc);
2019
2020         /* Start with SuperSpeed Default */
2021         dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2022
2023         dep = dwc->eps[0];
2024         ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
2025         if (ret) {
2026                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2027                 goto err0;
2028         }
2029
2030         dep = dwc->eps[1];
2031         ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
2032         if (ret) {
2033                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2034                 goto err1;
2035         }
2036
2037         /* begin to receive SETUP packets */
2038         dwc->ep0state = EP0_SETUP_PHASE;
2039         dwc->link_state = DWC3_LINK_STATE_SS_DIS;
2040         dwc3_ep0_out_start(dwc);
2041
2042         dwc3_gadget_enable_irq(dwc);
2043
2044         return 0;
2045
2046 err1:
2047         __dwc3_gadget_ep_disable(dwc->eps[0]);
2048
2049 err0:
2050         return ret;
2051 }
2052
2053 static int dwc3_gadget_start(struct usb_gadget *g,
2054                 struct usb_gadget_driver *driver)
2055 {
2056         struct dwc3             *dwc = gadget_to_dwc(g);
2057         unsigned long           flags;
2058         int                     ret = 0;
2059         int                     irq;
2060
2061         irq = dwc->irq_gadget;
2062         ret = request_threaded_irq(irq, dwc3_interrupt, dwc3_thread_interrupt,
2063                         IRQF_SHARED, "dwc3", dwc->ev_buf);
2064         if (ret) {
2065                 dev_err(dwc->dev, "failed to request irq #%d --> %d\n",
2066                                 irq, ret);
2067                 goto err0;
2068         }
2069
2070         spin_lock_irqsave(&dwc->lock, flags);
2071         if (dwc->gadget_driver) {
2072                 dev_err(dwc->dev, "%s is already bound to %s\n",
2073                                 dwc->gadget.name,
2074                                 dwc->gadget_driver->driver.name);
2075                 ret = -EBUSY;
2076                 goto err1;
2077         }
2078
2079         dwc->gadget_driver      = driver;
2080
2081         if (pm_runtime_active(dwc->dev))
2082                 __dwc3_gadget_start(dwc);
2083
2084         spin_unlock_irqrestore(&dwc->lock, flags);
2085
2086         return 0;
2087
2088 err1:
2089         spin_unlock_irqrestore(&dwc->lock, flags);
2090         free_irq(irq, dwc);
2091
2092 err0:
2093         return ret;
2094 }
2095
2096 static void __dwc3_gadget_stop(struct dwc3 *dwc)
2097 {
2098         dwc3_gadget_disable_irq(dwc);
2099         __dwc3_gadget_ep_disable(dwc->eps[0]);
2100         __dwc3_gadget_ep_disable(dwc->eps[1]);
2101 }
2102
2103 static int dwc3_gadget_stop(struct usb_gadget *g)
2104 {
2105         struct dwc3             *dwc = gadget_to_dwc(g);
2106         unsigned long           flags;
2107
2108         spin_lock_irqsave(&dwc->lock, flags);
2109
2110         if (pm_runtime_suspended(dwc->dev))
2111                 goto out;
2112
2113         __dwc3_gadget_stop(dwc);
2114
2115 out:
2116         dwc->gadget_driver      = NULL;
2117         spin_unlock_irqrestore(&dwc->lock, flags);
2118
2119         free_irq(dwc->irq_gadget, dwc->ev_buf);
2120
2121         return 0;
2122 }
2123
2124 static void dwc3_gadget_config_params(struct usb_gadget *g,
2125                                       struct usb_dcd_config_params *params)
2126 {
2127         struct dwc3             *dwc = gadget_to_dwc(g);
2128
2129         params->besl_baseline = USB_DEFAULT_BESL_UNSPECIFIED;
2130         params->besl_deep = USB_DEFAULT_BESL_UNSPECIFIED;
2131
2132         /* Recommended BESL */
2133         if (!dwc->dis_enblslpm_quirk) {
2134                 /*
2135                  * If the recommended BESL baseline is 0 or if the BESL deep is
2136                  * less than 2, Microsoft's Windows 10 host usb stack will issue
2137                  * a usb reset immediately after it receives the extended BOS
2138                  * descriptor and the enumeration will fail. To maintain
2139                  * compatibility with the Windows' usb stack, let's set the
2140                  * recommended BESL baseline to 1 and clamp the BESL deep to be
2141                  * within 2 to 15.
2142                  */
2143                 params->besl_baseline = 1;
2144                 if (dwc->is_utmi_l1_suspend)
2145                         params->besl_deep =
2146                                 clamp_t(u8, dwc->hird_threshold, 2, 15);
2147         }
2148
2149         /* U1 Device exit Latency */
2150         if (dwc->dis_u1_entry_quirk)
2151                 params->bU1devExitLat = 0;
2152         else
2153                 params->bU1devExitLat = DWC3_DEFAULT_U1_DEV_EXIT_LAT;
2154
2155         /* U2 Device exit Latency */
2156         if (dwc->dis_u2_entry_quirk)
2157                 params->bU2DevExitLat = 0;
2158         else
2159                 params->bU2DevExitLat =
2160                                 cpu_to_le16(DWC3_DEFAULT_U2_DEV_EXIT_LAT);
2161 }
2162
2163 static void dwc3_gadget_set_speed(struct usb_gadget *g,
2164                                   enum usb_device_speed speed)
2165 {
2166         struct dwc3             *dwc = gadget_to_dwc(g);
2167         unsigned long           flags;
2168         u32                     reg;
2169
2170         spin_lock_irqsave(&dwc->lock, flags);
2171         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2172         reg &= ~(DWC3_DCFG_SPEED_MASK);
2173
2174         /*
2175          * WORKAROUND: DWC3 revision < 2.20a have an issue
2176          * which would cause metastability state on Run/Stop
2177          * bit if we try to force the IP to USB2-only mode.
2178          *
2179          * Because of that, we cannot configure the IP to any
2180          * speed other than the SuperSpeed
2181          *
2182          * Refers to:
2183          *
2184          * STAR#9000525659: Clock Domain Crossing on DCTL in
2185          * USB 2.0 Mode
2186          */
2187         if (dwc->revision < DWC3_REVISION_220A &&
2188             !dwc->dis_metastability_quirk) {
2189                 reg |= DWC3_DCFG_SUPERSPEED;
2190         } else {
2191                 switch (speed) {
2192                 case USB_SPEED_LOW:
2193                         reg |= DWC3_DCFG_LOWSPEED;
2194                         break;
2195                 case USB_SPEED_FULL:
2196                         reg |= DWC3_DCFG_FULLSPEED;
2197                         break;
2198                 case USB_SPEED_HIGH:
2199                         reg |= DWC3_DCFG_HIGHSPEED;
2200                         break;
2201                 case USB_SPEED_SUPER:
2202                         reg |= DWC3_DCFG_SUPERSPEED;
2203                         break;
2204                 case USB_SPEED_SUPER_PLUS:
2205                         if (dwc3_is_usb31(dwc))
2206                                 reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2207                         else
2208                                 reg |= DWC3_DCFG_SUPERSPEED;
2209                         break;
2210                 default:
2211                         dev_err(dwc->dev, "invalid speed (%d)\n", speed);
2212
2213                         if (dwc->revision & DWC3_REVISION_IS_DWC31)
2214                                 reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2215                         else
2216                                 reg |= DWC3_DCFG_SUPERSPEED;
2217                 }
2218         }
2219         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2220
2221         spin_unlock_irqrestore(&dwc->lock, flags);
2222 }
2223
2224 static const struct usb_gadget_ops dwc3_gadget_ops = {
2225         .get_frame              = dwc3_gadget_get_frame,
2226         .wakeup                 = dwc3_gadget_wakeup,
2227         .set_selfpowered        = dwc3_gadget_set_selfpowered,
2228         .pullup                 = dwc3_gadget_pullup,
2229         .udc_start              = dwc3_gadget_start,
2230         .udc_stop               = dwc3_gadget_stop,
2231         .udc_set_speed          = dwc3_gadget_set_speed,
2232         .get_config_params      = dwc3_gadget_config_params,
2233 };
2234
2235 /* -------------------------------------------------------------------------- */
2236
2237 static int dwc3_gadget_init_control_endpoint(struct dwc3_ep *dep)
2238 {
2239         struct dwc3 *dwc = dep->dwc;
2240
2241         usb_ep_set_maxpacket_limit(&dep->endpoint, 512);
2242         dep->endpoint.maxburst = 1;
2243         dep->endpoint.ops = &dwc3_gadget_ep0_ops;
2244         if (!dep->direction)
2245                 dwc->gadget.ep0 = &dep->endpoint;
2246
2247         dep->endpoint.caps.type_control = true;
2248
2249         return 0;
2250 }
2251
2252 static int dwc3_gadget_init_in_endpoint(struct dwc3_ep *dep)
2253 {
2254         struct dwc3 *dwc = dep->dwc;
2255         int mdwidth;
2256         int size;
2257
2258         mdwidth = DWC3_MDWIDTH(dwc->hwparams.hwparams0);
2259         /* MDWIDTH is represented in bits, we need it in bytes */
2260         mdwidth /= 8;
2261
2262         size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1));
2263         if (dwc3_is_usb31(dwc))
2264                 size = DWC31_GTXFIFOSIZ_TXFDEP(size);
2265         else
2266                 size = DWC3_GTXFIFOSIZ_TXFDEP(size);
2267
2268         /* FIFO Depth is in MDWDITH bytes. Multiply */
2269         size *= mdwidth;
2270
2271         /*
2272          * To meet performance requirement, a minimum TxFIFO size of 3x
2273          * MaxPacketSize is recommended for endpoints that support burst and a
2274          * minimum TxFIFO size of 2x MaxPacketSize for endpoints that don't
2275          * support burst. Use those numbers and we can calculate the max packet
2276          * limit as below.
2277          */
2278         if (dwc->maximum_speed >= USB_SPEED_SUPER)
2279                 size /= 3;
2280         else
2281                 size /= 2;
2282
2283         usb_ep_set_maxpacket_limit(&dep->endpoint, size);
2284
2285         dep->endpoint.max_streams = 15;
2286         dep->endpoint.ops = &dwc3_gadget_ep_ops;
2287         list_add_tail(&dep->endpoint.ep_list,
2288                         &dwc->gadget.ep_list);
2289         dep->endpoint.caps.type_iso = true;
2290         dep->endpoint.caps.type_bulk = true;
2291         dep->endpoint.caps.type_int = true;
2292
2293         return dwc3_alloc_trb_pool(dep);
2294 }
2295
2296 static int dwc3_gadget_init_out_endpoint(struct dwc3_ep *dep)
2297 {
2298         struct dwc3 *dwc = dep->dwc;
2299         int mdwidth;
2300         int size;
2301
2302         mdwidth = DWC3_MDWIDTH(dwc->hwparams.hwparams0);
2303
2304         /* MDWIDTH is represented in bits, convert to bytes */
2305         mdwidth /= 8;
2306
2307         /* All OUT endpoints share a single RxFIFO space */
2308         size = dwc3_readl(dwc->regs, DWC3_GRXFIFOSIZ(0));
2309         if (dwc3_is_usb31(dwc))
2310                 size = DWC31_GRXFIFOSIZ_RXFDEP(size);
2311         else
2312                 size = DWC3_GRXFIFOSIZ_RXFDEP(size);
2313
2314         /* FIFO depth is in MDWDITH bytes */
2315         size *= mdwidth;
2316
2317         /*
2318          * To meet performance requirement, a minimum recommended RxFIFO size
2319          * is defined as follow:
2320          * RxFIFO size >= (3 x MaxPacketSize) +
2321          * (3 x 8 bytes setup packets size) + (16 bytes clock crossing margin)
2322          *
2323          * Then calculate the max packet limit as below.
2324          */
2325         size -= (3 * 8) + 16;
2326         if (size < 0)
2327                 size = 0;
2328         else
2329                 size /= 3;
2330
2331         usb_ep_set_maxpacket_limit(&dep->endpoint, size);
2332         dep->endpoint.max_streams = 15;
2333         dep->endpoint.ops = &dwc3_gadget_ep_ops;
2334         list_add_tail(&dep->endpoint.ep_list,
2335                         &dwc->gadget.ep_list);
2336         dep->endpoint.caps.type_iso = true;
2337         dep->endpoint.caps.type_bulk = true;
2338         dep->endpoint.caps.type_int = true;
2339
2340         return dwc3_alloc_trb_pool(dep);
2341 }
2342
2343 static int dwc3_gadget_init_endpoint(struct dwc3 *dwc, u8 epnum)
2344 {
2345         struct dwc3_ep                  *dep;
2346         bool                            direction = epnum & 1;
2347         int                             ret;
2348         u8                              num = epnum >> 1;
2349
2350         dep = kzalloc(sizeof(*dep), GFP_KERNEL);
2351         if (!dep)
2352                 return -ENOMEM;
2353
2354         dep->dwc = dwc;
2355         dep->number = epnum;
2356         dep->direction = direction;
2357         dep->regs = dwc->regs + DWC3_DEP_BASE(epnum);
2358         dwc->eps[epnum] = dep;
2359         dep->combo_num = 0;
2360         dep->start_cmd_status = 0;
2361
2362         snprintf(dep->name, sizeof(dep->name), "ep%u%s", num,
2363                         direction ? "in" : "out");
2364
2365         dep->endpoint.name = dep->name;
2366
2367         if (!(dep->number > 1)) {
2368                 dep->endpoint.desc = &dwc3_gadget_ep0_desc;
2369                 dep->endpoint.comp_desc = NULL;
2370         }
2371
2372         if (num == 0)
2373                 ret = dwc3_gadget_init_control_endpoint(dep);
2374         else if (direction)
2375                 ret = dwc3_gadget_init_in_endpoint(dep);
2376         else
2377                 ret = dwc3_gadget_init_out_endpoint(dep);
2378
2379         if (ret)
2380                 return ret;
2381
2382         dep->endpoint.caps.dir_in = direction;
2383         dep->endpoint.caps.dir_out = !direction;
2384
2385         INIT_LIST_HEAD(&dep->pending_list);
2386         INIT_LIST_HEAD(&dep->started_list);
2387         INIT_LIST_HEAD(&dep->cancelled_list);
2388
2389         return 0;
2390 }
2391
2392 static int dwc3_gadget_init_endpoints(struct dwc3 *dwc, u8 total)
2393 {
2394         u8                              epnum;
2395
2396         INIT_LIST_HEAD(&dwc->gadget.ep_list);
2397
2398         for (epnum = 0; epnum < total; epnum++) {
2399                 int                     ret;
2400
2401                 ret = dwc3_gadget_init_endpoint(dwc, epnum);
2402                 if (ret)
2403                         return ret;
2404         }
2405
2406         return 0;
2407 }
2408
2409 static void dwc3_gadget_free_endpoints(struct dwc3 *dwc)
2410 {
2411         struct dwc3_ep                  *dep;
2412         u8                              epnum;
2413
2414         for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2415                 dep = dwc->eps[epnum];
2416                 if (!dep)
2417                         continue;
2418                 /*
2419                  * Physical endpoints 0 and 1 are special; they form the
2420                  * bi-directional USB endpoint 0.
2421                  *
2422                  * For those two physical endpoints, we don't allocate a TRB
2423                  * pool nor do we add them the endpoints list. Due to that, we
2424                  * shouldn't do these two operations otherwise we would end up
2425                  * with all sorts of bugs when removing dwc3.ko.
2426                  */
2427                 if (epnum != 0 && epnum != 1) {
2428                         dwc3_free_trb_pool(dep);
2429                         list_del(&dep->endpoint.ep_list);
2430                 }
2431
2432                 kfree(dep);
2433         }
2434 }
2435
2436 /* -------------------------------------------------------------------------- */
2437
2438 static int dwc3_gadget_ep_reclaim_completed_trb(struct dwc3_ep *dep,
2439                 struct dwc3_request *req, struct dwc3_trb *trb,
2440                 const struct dwc3_event_depevt *event, int status, int chain)
2441 {
2442         unsigned int            count;
2443
2444         dwc3_ep_inc_deq(dep);
2445
2446         trace_dwc3_complete_trb(dep, trb);
2447         req->num_trbs--;
2448
2449         /*
2450          * If we're in the middle of series of chained TRBs and we
2451          * receive a short transfer along the way, DWC3 will skip
2452          * through all TRBs including the last TRB in the chain (the
2453          * where CHN bit is zero. DWC3 will also avoid clearing HWO
2454          * bit and SW has to do it manually.
2455          *
2456          * We're going to do that here to avoid problems of HW trying
2457          * to use bogus TRBs for transfers.
2458          */
2459         if (chain && (trb->ctrl & DWC3_TRB_CTRL_HWO))
2460                 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
2461
2462         /*
2463          * For isochronous transfers, the first TRB in a service interval must
2464          * have the Isoc-First type. Track and report its interval frame number.
2465          */
2466         if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
2467             (trb->ctrl & DWC3_TRBCTL_ISOCHRONOUS_FIRST)) {
2468                 unsigned int frame_number;
2469
2470                 frame_number = DWC3_TRB_CTRL_GET_SID_SOFN(trb->ctrl);
2471                 frame_number &= ~(dep->interval - 1);
2472                 req->request.frame_number = frame_number;
2473         }
2474
2475         /*
2476          * If we're dealing with unaligned size OUT transfer, we will be left
2477          * with one TRB pending in the ring. We need to manually clear HWO bit
2478          * from that TRB.
2479          */
2480
2481         if (req->needs_extra_trb && !(trb->ctrl & DWC3_TRB_CTRL_CHN)) {
2482                 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
2483                 return 1;
2484         }
2485
2486         count = trb->size & DWC3_TRB_SIZE_MASK;
2487         req->remaining += count;
2488
2489         if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN)
2490                 return 1;
2491
2492         if (event->status & DEPEVT_STATUS_SHORT && !chain)
2493                 return 1;
2494
2495         if ((trb->ctrl & DWC3_TRB_CTRL_IOC) ||
2496             (trb->ctrl & DWC3_TRB_CTRL_LST))
2497                 return 1;
2498
2499         return 0;
2500 }
2501
2502 static int dwc3_gadget_ep_reclaim_trb_sg(struct dwc3_ep *dep,
2503                 struct dwc3_request *req, const struct dwc3_event_depevt *event,
2504                 int status)
2505 {
2506         struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];
2507         struct scatterlist *sg = req->sg;
2508         struct scatterlist *s;
2509         unsigned int pending = req->num_pending_sgs;
2510         unsigned int i;
2511         int ret = 0;
2512
2513         for_each_sg(sg, s, pending, i) {
2514                 trb = &dep->trb_pool[dep->trb_dequeue];
2515
2516                 if (trb->ctrl & DWC3_TRB_CTRL_HWO)
2517                         break;
2518
2519                 req->sg = sg_next(s);
2520                 req->num_pending_sgs--;
2521
2522                 ret = dwc3_gadget_ep_reclaim_completed_trb(dep, req,
2523                                 trb, event, status, true);
2524                 if (ret)
2525                         break;
2526         }
2527
2528         return ret;
2529 }
2530
2531 static int dwc3_gadget_ep_reclaim_trb_linear(struct dwc3_ep *dep,
2532                 struct dwc3_request *req, const struct dwc3_event_depevt *event,
2533                 int status)
2534 {
2535         struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];
2536
2537         return dwc3_gadget_ep_reclaim_completed_trb(dep, req, trb,
2538                         event, status, false);
2539 }
2540
2541 static bool dwc3_gadget_ep_request_completed(struct dwc3_request *req)
2542 {
2543         return req->num_pending_sgs == 0;
2544 }
2545
2546 static int dwc3_gadget_ep_cleanup_completed_request(struct dwc3_ep *dep,
2547                 const struct dwc3_event_depevt *event,
2548                 struct dwc3_request *req, int status)
2549 {
2550         int ret;
2551
2552         if (req->num_pending_sgs)
2553                 ret = dwc3_gadget_ep_reclaim_trb_sg(dep, req, event,
2554                                 status);
2555         else
2556                 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
2557                                 status);
2558
2559         if (req->needs_extra_trb) {
2560                 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
2561                                 status);
2562                 req->needs_extra_trb = false;
2563         }
2564
2565         req->request.actual = req->request.length - req->remaining;
2566
2567         if (!dwc3_gadget_ep_request_completed(req)) {
2568                 __dwc3_gadget_kick_transfer(dep);
2569                 goto out;
2570         }
2571
2572         dwc3_gadget_giveback(dep, req, status);
2573
2574 out:
2575         return ret;
2576 }
2577
2578 static void dwc3_gadget_ep_cleanup_completed_requests(struct dwc3_ep *dep,
2579                 const struct dwc3_event_depevt *event, int status)
2580 {
2581         struct dwc3_request     *req;
2582         struct dwc3_request     *tmp;
2583
2584         list_for_each_entry_safe(req, tmp, &dep->started_list, list) {
2585                 int ret;
2586
2587                 ret = dwc3_gadget_ep_cleanup_completed_request(dep, event,
2588                                 req, status);
2589                 if (ret)
2590                         break;
2591         }
2592 }
2593
2594 static void dwc3_gadget_endpoint_frame_from_event(struct dwc3_ep *dep,
2595                 const struct dwc3_event_depevt *event)
2596 {
2597         dep->frame_number = event->parameters;
2598 }
2599
2600 static void dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep *dep,
2601                 const struct dwc3_event_depevt *event)
2602 {
2603         struct dwc3             *dwc = dep->dwc;
2604         unsigned                status = 0;
2605         bool                    stop = false;
2606
2607         dwc3_gadget_endpoint_frame_from_event(dep, event);
2608
2609         if (event->status & DEPEVT_STATUS_BUSERR)
2610                 status = -ECONNRESET;
2611
2612         if (event->status & DEPEVT_STATUS_MISSED_ISOC) {
2613                 status = -EXDEV;
2614
2615                 if (list_empty(&dep->started_list))
2616                         stop = true;
2617         }
2618
2619         dwc3_gadget_ep_cleanup_completed_requests(dep, event, status);
2620
2621         if (stop)
2622                 dwc3_stop_active_transfer(dep, true, true);
2623
2624         /*
2625          * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround.
2626          * See dwc3_gadget_linksts_change_interrupt() for 1st half.
2627          */
2628         if (dwc->revision < DWC3_REVISION_183A) {
2629                 u32             reg;
2630                 int             i;
2631
2632                 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
2633                         dep = dwc->eps[i];
2634
2635                         if (!(dep->flags & DWC3_EP_ENABLED))
2636                                 continue;
2637
2638                         if (!list_empty(&dep->started_list))
2639                                 return;
2640                 }
2641
2642                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2643                 reg |= dwc->u1u2;
2644                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2645
2646                 dwc->u1u2 = 0;
2647         }
2648 }
2649
2650 static void dwc3_gadget_endpoint_transfer_not_ready(struct dwc3_ep *dep,
2651                 const struct dwc3_event_depevt *event)
2652 {
2653         dwc3_gadget_endpoint_frame_from_event(dep, event);
2654         (void) __dwc3_gadget_start_isoc(dep);
2655 }
2656
2657 static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
2658                 const struct dwc3_event_depevt *event)
2659 {
2660         struct dwc3_ep          *dep;
2661         u8                      epnum = event->endpoint_number;
2662         u8                      cmd;
2663
2664         dep = dwc->eps[epnum];
2665
2666         if (!(dep->flags & DWC3_EP_ENABLED)) {
2667                 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED))
2668                         return;
2669
2670                 /* Handle only EPCMDCMPLT when EP disabled */
2671                 if (event->endpoint_event != DWC3_DEPEVT_EPCMDCMPLT)
2672                         return;
2673         }
2674
2675         if (epnum == 0 || epnum == 1) {
2676                 dwc3_ep0_interrupt(dwc, event);
2677                 return;
2678         }
2679
2680         switch (event->endpoint_event) {
2681         case DWC3_DEPEVT_XFERINPROGRESS:
2682                 dwc3_gadget_endpoint_transfer_in_progress(dep, event);
2683                 break;
2684         case DWC3_DEPEVT_XFERNOTREADY:
2685                 dwc3_gadget_endpoint_transfer_not_ready(dep, event);
2686                 break;
2687         case DWC3_DEPEVT_EPCMDCMPLT:
2688                 cmd = DEPEVT_PARAMETER_CMD(event->parameters);
2689
2690                 if (cmd == DWC3_DEPCMD_ENDTRANSFER) {
2691                         dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
2692                         dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
2693                         dwc3_gadget_ep_cleanup_cancelled_requests(dep);
2694                         if ((dep->flags & DWC3_EP_DELAY_START) &&
2695                             !usb_endpoint_xfer_isoc(dep->endpoint.desc))
2696                                 __dwc3_gadget_kick_transfer(dep);
2697
2698                         dep->flags &= ~DWC3_EP_DELAY_START;
2699                 }
2700                 break;
2701         case DWC3_DEPEVT_STREAMEVT:
2702         case DWC3_DEPEVT_XFERCOMPLETE:
2703         case DWC3_DEPEVT_RXTXFIFOEVT:
2704                 break;
2705         }
2706 }
2707
2708 static void dwc3_disconnect_gadget(struct dwc3 *dwc)
2709 {
2710         if (dwc->gadget_driver && dwc->gadget_driver->disconnect) {
2711                 spin_unlock(&dwc->lock);
2712                 dwc->gadget_driver->disconnect(&dwc->gadget);
2713                 spin_lock(&dwc->lock);
2714         }
2715 }
2716
2717 static void dwc3_suspend_gadget(struct dwc3 *dwc)
2718 {
2719         if (dwc->gadget_driver && dwc->gadget_driver->suspend) {
2720                 spin_unlock(&dwc->lock);
2721                 dwc->gadget_driver->suspend(&dwc->gadget);
2722                 spin_lock(&dwc->lock);
2723         }
2724 }
2725
2726 static void dwc3_resume_gadget(struct dwc3 *dwc)
2727 {
2728         if (dwc->gadget_driver && dwc->gadget_driver->resume) {
2729                 spin_unlock(&dwc->lock);
2730                 dwc->gadget_driver->resume(&dwc->gadget);
2731                 spin_lock(&dwc->lock);
2732         }
2733 }
2734
2735 static void dwc3_reset_gadget(struct dwc3 *dwc)
2736 {
2737         if (!dwc->gadget_driver)
2738                 return;
2739
2740         if (dwc->gadget.speed != USB_SPEED_UNKNOWN) {
2741                 spin_unlock(&dwc->lock);
2742                 usb_gadget_udc_reset(&dwc->gadget, dwc->gadget_driver);
2743                 spin_lock(&dwc->lock);
2744         }
2745 }
2746
2747 static void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force,
2748         bool interrupt)
2749 {
2750         struct dwc3_gadget_ep_cmd_params params;
2751         u32 cmd;
2752         int ret;
2753
2754         if (!(dep->flags & DWC3_EP_TRANSFER_STARTED) ||
2755             (dep->flags & DWC3_EP_END_TRANSFER_PENDING))
2756                 return;
2757
2758         /*
2759          * NOTICE: We are violating what the Databook says about the
2760          * EndTransfer command. Ideally we would _always_ wait for the
2761          * EndTransfer Command Completion IRQ, but that's causing too
2762          * much trouble synchronizing between us and gadget driver.
2763          *
2764          * We have discussed this with the IP Provider and it was
2765          * suggested to giveback all requests here.
2766          *
2767          * Note also that a similar handling was tested by Synopsys
2768          * (thanks a lot Paul) and nothing bad has come out of it.
2769          * In short, what we're doing is issuing EndTransfer with
2770          * CMDIOC bit set and delay kicking transfer until the
2771          * EndTransfer command had completed.
2772          *
2773          * As of IP version 3.10a of the DWC_usb3 IP, the controller
2774          * supports a mode to work around the above limitation. The
2775          * software can poll the CMDACT bit in the DEPCMD register
2776          * after issuing a EndTransfer command. This mode is enabled
2777          * by writing GUCTL2[14]. This polling is already done in the
2778          * dwc3_send_gadget_ep_cmd() function so if the mode is
2779          * enabled, the EndTransfer command will have completed upon
2780          * returning from this function.
2781          *
2782          * This mode is NOT available on the DWC_usb31 IP.
2783          */
2784
2785         cmd = DWC3_DEPCMD_ENDTRANSFER;
2786         cmd |= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0;
2787         cmd |= interrupt ? DWC3_DEPCMD_CMDIOC : 0;
2788         cmd |= DWC3_DEPCMD_PARAM(dep->resource_index);
2789         memset(&params, 0, sizeof(params));
2790         ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
2791         WARN_ON_ONCE(ret);
2792         dep->resource_index = 0;
2793
2794         if (!interrupt)
2795                 dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
2796         else
2797                 dep->flags |= DWC3_EP_END_TRANSFER_PENDING;
2798 }
2799
2800 static void dwc3_clear_stall_all_ep(struct dwc3 *dwc)
2801 {
2802         u32 epnum;
2803
2804         for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2805                 struct dwc3_ep *dep;
2806                 int ret;
2807
2808                 dep = dwc->eps[epnum];
2809                 if (!dep)
2810                         continue;
2811
2812                 if (!(dep->flags & DWC3_EP_STALL))
2813                         continue;
2814
2815                 dep->flags &= ~DWC3_EP_STALL;
2816
2817                 ret = dwc3_send_clear_stall_ep_cmd(dep);
2818                 WARN_ON_ONCE(ret);
2819         }
2820 }
2821
2822 static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc)
2823 {
2824         int                     reg;
2825
2826         dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RX_DET);
2827
2828         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2829         reg &= ~DWC3_DCTL_INITU1ENA;
2830         reg &= ~DWC3_DCTL_INITU2ENA;
2831         dwc3_gadget_dctl_write_safe(dwc, reg);
2832
2833         dwc3_disconnect_gadget(dwc);
2834
2835         dwc->gadget.speed = USB_SPEED_UNKNOWN;
2836         dwc->setup_packet_pending = false;
2837         usb_gadget_set_state(&dwc->gadget, USB_STATE_NOTATTACHED);
2838
2839         dwc->connected = false;
2840 }
2841
2842 static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
2843 {
2844         u32                     reg;
2845
2846         dwc->connected = true;
2847
2848         /*
2849          * WORKAROUND: DWC3 revisions <1.88a have an issue which
2850          * would cause a missing Disconnect Event if there's a
2851          * pending Setup Packet in the FIFO.
2852          *
2853          * There's no suggested workaround on the official Bug
2854          * report, which states that "unless the driver/application
2855          * is doing any special handling of a disconnect event,
2856          * there is no functional issue".
2857          *
2858          * Unfortunately, it turns out that we _do_ some special
2859          * handling of a disconnect event, namely complete all
2860          * pending transfers, notify gadget driver of the
2861          * disconnection, and so on.
2862          *
2863          * Our suggested workaround is to follow the Disconnect
2864          * Event steps here, instead, based on a setup_packet_pending
2865          * flag. Such flag gets set whenever we have a SETUP_PENDING
2866          * status for EP0 TRBs and gets cleared on XferComplete for the
2867          * same endpoint.
2868          *
2869          * Refers to:
2870          *
2871          * STAR#9000466709: RTL: Device : Disconnect event not
2872          * generated if setup packet pending in FIFO
2873          */
2874         if (dwc->revision < DWC3_REVISION_188A) {
2875                 if (dwc->setup_packet_pending)
2876                         dwc3_gadget_disconnect_interrupt(dwc);
2877         }
2878
2879         dwc3_reset_gadget(dwc);
2880
2881         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2882         reg &= ~DWC3_DCTL_TSTCTRL_MASK;
2883         dwc3_gadget_dctl_write_safe(dwc, reg);
2884         dwc->test_mode = false;
2885         dwc3_clear_stall_all_ep(dwc);
2886
2887         /* Reset device address to zero */
2888         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2889         reg &= ~(DWC3_DCFG_DEVADDR_MASK);
2890         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2891 }
2892
2893 static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
2894 {
2895         struct dwc3_ep          *dep;
2896         int                     ret;
2897         u32                     reg;
2898         u8                      speed;
2899
2900         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2901         speed = reg & DWC3_DSTS_CONNECTSPD;
2902         dwc->speed = speed;
2903
2904         /*
2905          * RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed
2906          * each time on Connect Done.
2907          *
2908          * Currently we always use the reset value. If any platform
2909          * wants to set this to a different value, we need to add a
2910          * setting and update GCTL.RAMCLKSEL here.
2911          */
2912
2913         switch (speed) {
2914         case DWC3_DSTS_SUPERSPEED_PLUS:
2915                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2916                 dwc->gadget.ep0->maxpacket = 512;
2917                 dwc->gadget.speed = USB_SPEED_SUPER_PLUS;
2918                 break;
2919         case DWC3_DSTS_SUPERSPEED:
2920                 /*
2921                  * WORKAROUND: DWC3 revisions <1.90a have an issue which
2922                  * would cause a missing USB3 Reset event.
2923                  *
2924                  * In such situations, we should force a USB3 Reset
2925                  * event by calling our dwc3_gadget_reset_interrupt()
2926                  * routine.
2927                  *
2928                  * Refers to:
2929                  *
2930                  * STAR#9000483510: RTL: SS : USB3 reset event may
2931                  * not be generated always when the link enters poll
2932                  */
2933                 if (dwc->revision < DWC3_REVISION_190A)
2934                         dwc3_gadget_reset_interrupt(dwc);
2935
2936                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2937                 dwc->gadget.ep0->maxpacket = 512;
2938                 dwc->gadget.speed = USB_SPEED_SUPER;
2939                 break;
2940         case DWC3_DSTS_HIGHSPEED:
2941                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
2942                 dwc->gadget.ep0->maxpacket = 64;
2943                 dwc->gadget.speed = USB_SPEED_HIGH;
2944                 break;
2945         case DWC3_DSTS_FULLSPEED:
2946                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
2947                 dwc->gadget.ep0->maxpacket = 64;
2948                 dwc->gadget.speed = USB_SPEED_FULL;
2949                 break;
2950         case DWC3_DSTS_LOWSPEED:
2951                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(8);
2952                 dwc->gadget.ep0->maxpacket = 8;
2953                 dwc->gadget.speed = USB_SPEED_LOW;
2954                 break;
2955         }
2956
2957         dwc->eps[1]->endpoint.maxpacket = dwc->gadget.ep0->maxpacket;
2958
2959         /* Enable USB2 LPM Capability */
2960
2961         if ((dwc->revision > DWC3_REVISION_194A) &&
2962             (speed != DWC3_DSTS_SUPERSPEED) &&
2963             (speed != DWC3_DSTS_SUPERSPEED_PLUS)) {
2964                 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2965                 reg |= DWC3_DCFG_LPM_CAP;
2966                 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2967
2968                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2969                 reg &= ~(DWC3_DCTL_HIRD_THRES_MASK | DWC3_DCTL_L1_HIBER_EN);
2970
2971                 reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold |
2972                                             (dwc->is_utmi_l1_suspend << 4));
2973
2974                 /*
2975                  * When dwc3 revisions >= 2.40a, LPM Erratum is enabled and
2976                  * DCFG.LPMCap is set, core responses with an ACK and the
2977                  * BESL value in the LPM token is less than or equal to LPM
2978                  * NYET threshold.
2979                  */
2980                 WARN_ONCE(dwc->revision < DWC3_REVISION_240A
2981                                 && dwc->has_lpm_erratum,
2982                                 "LPM Erratum not available on dwc3 revisions < 2.40a\n");
2983
2984                 if (dwc->has_lpm_erratum && dwc->revision >= DWC3_REVISION_240A)
2985                         reg |= DWC3_DCTL_NYET_THRES(dwc->lpm_nyet_threshold);
2986
2987                 dwc3_gadget_dctl_write_safe(dwc, reg);
2988         } else {
2989                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2990                 reg &= ~DWC3_DCTL_HIRD_THRES_MASK;
2991                 dwc3_gadget_dctl_write_safe(dwc, reg);
2992         }
2993
2994         dep = dwc->eps[0];
2995         ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
2996         if (ret) {
2997                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2998                 return;
2999         }
3000
3001         dep = dwc->eps[1];
3002         ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
3003         if (ret) {
3004                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
3005                 return;
3006         }
3007
3008         /*
3009          * Configure PHY via GUSB3PIPECTLn if required.
3010          *
3011          * Update GTXFIFOSIZn
3012          *
3013          * In both cases reset values should be sufficient.
3014          */
3015 }
3016
3017 static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc)
3018 {
3019         /*
3020          * TODO take core out of low power mode when that's
3021          * implemented.
3022          */
3023
3024         if (dwc->gadget_driver && dwc->gadget_driver->resume) {
3025                 spin_unlock(&dwc->lock);
3026                 dwc->gadget_driver->resume(&dwc->gadget);
3027                 spin_lock(&dwc->lock);
3028         }
3029 }
3030
3031 static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
3032                 unsigned int evtinfo)
3033 {
3034         enum dwc3_link_state    next = evtinfo & DWC3_LINK_STATE_MASK;
3035         unsigned int            pwropt;
3036
3037         /*
3038          * WORKAROUND: DWC3 < 2.50a have an issue when configured without
3039          * Hibernation mode enabled which would show up when device detects
3040          * host-initiated U3 exit.
3041          *
3042          * In that case, device will generate a Link State Change Interrupt
3043          * from U3 to RESUME which is only necessary if Hibernation is
3044          * configured in.
3045          *
3046          * There are no functional changes due to such spurious event and we
3047          * just need to ignore it.
3048          *
3049          * Refers to:
3050          *
3051          * STAR#9000570034 RTL: SS Resume event generated in non-Hibernation
3052          * operational mode
3053          */
3054         pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1);
3055         if ((dwc->revision < DWC3_REVISION_250A) &&
3056                         (pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) {
3057                 if ((dwc->link_state == DWC3_LINK_STATE_U3) &&
3058                                 (next == DWC3_LINK_STATE_RESUME)) {
3059                         return;
3060                 }
3061         }
3062
3063         /*
3064          * WORKAROUND: DWC3 Revisions <1.83a have an issue which, depending
3065          * on the link partner, the USB session might do multiple entry/exit
3066          * of low power states before a transfer takes place.
3067          *
3068          * Due to this problem, we might experience lower throughput. The
3069          * suggested workaround is to disable DCTL[12:9] bits if we're
3070          * transitioning from U1/U2 to U0 and enable those bits again
3071          * after a transfer completes and there are no pending transfers
3072          * on any of the enabled endpoints.
3073          *
3074          * This is the first half of that workaround.
3075          *
3076          * Refers to:
3077          *
3078          * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us
3079          * core send LGO_Ux entering U0
3080          */
3081         if (dwc->revision < DWC3_REVISION_183A) {
3082                 if (next == DWC3_LINK_STATE_U0) {
3083                         u32     u1u2;
3084                         u32     reg;
3085
3086                         switch (dwc->link_state) {
3087                         case DWC3_LINK_STATE_U1:
3088                         case DWC3_LINK_STATE_U2:
3089                                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3090                                 u1u2 = reg & (DWC3_DCTL_INITU2ENA
3091                                                 | DWC3_DCTL_ACCEPTU2ENA
3092                                                 | DWC3_DCTL_INITU1ENA
3093                                                 | DWC3_DCTL_ACCEPTU1ENA);
3094
3095                                 if (!dwc->u1u2)
3096                                         dwc->u1u2 = reg & u1u2;
3097
3098                                 reg &= ~u1u2;
3099
3100                                 dwc3_gadget_dctl_write_safe(dwc, reg);
3101                                 break;
3102                         default:
3103                                 /* do nothing */
3104                                 break;
3105                         }
3106                 }
3107         }
3108
3109         switch (next) {
3110         case DWC3_LINK_STATE_U1:
3111                 if (dwc->speed == USB_SPEED_SUPER)
3112                         dwc3_suspend_gadget(dwc);
3113                 break;
3114         case DWC3_LINK_STATE_U2:
3115         case DWC3_LINK_STATE_U3:
3116                 dwc3_suspend_gadget(dwc);
3117                 break;
3118         case DWC3_LINK_STATE_RESUME:
3119                 dwc3_resume_gadget(dwc);
3120                 break;
3121         default:
3122                 /* do nothing */
3123                 break;
3124         }
3125
3126         dwc->link_state = next;
3127 }
3128
3129 static void dwc3_gadget_suspend_interrupt(struct dwc3 *dwc,
3130                                           unsigned int evtinfo)
3131 {
3132         enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
3133
3134         if (dwc->link_state != next && next == DWC3_LINK_STATE_U3)
3135                 dwc3_suspend_gadget(dwc);
3136
3137         dwc->link_state = next;
3138 }
3139
3140 static void dwc3_gadget_hibernation_interrupt(struct dwc3 *dwc,
3141                 unsigned int evtinfo)
3142 {
3143         unsigned int is_ss = evtinfo & BIT(4);
3144
3145         /*
3146          * WORKAROUND: DWC3 revison 2.20a with hibernation support
3147          * have a known issue which can cause USB CV TD.9.23 to fail
3148          * randomly.
3149          *
3150          * Because of this issue, core could generate bogus hibernation
3151          * events which SW needs to ignore.
3152          *
3153          * Refers to:
3154          *
3155          * STAR#9000546576: Device Mode Hibernation: Issue in USB 2.0
3156          * Device Fallback from SuperSpeed
3157          */
3158         if (is_ss ^ (dwc->speed == USB_SPEED_SUPER))
3159                 return;
3160
3161         /* enter hibernation here */
3162 }
3163
3164 static void dwc3_gadget_interrupt(struct dwc3 *dwc,
3165                 const struct dwc3_event_devt *event)
3166 {
3167         switch (event->type) {
3168         case DWC3_DEVICE_EVENT_DISCONNECT:
3169                 dwc3_gadget_disconnect_interrupt(dwc);
3170                 break;
3171         case DWC3_DEVICE_EVENT_RESET:
3172                 dwc3_gadget_reset_interrupt(dwc);
3173                 break;
3174         case DWC3_DEVICE_EVENT_CONNECT_DONE:
3175                 dwc3_gadget_conndone_interrupt(dwc);
3176                 break;
3177         case DWC3_DEVICE_EVENT_WAKEUP:
3178                 dwc3_gadget_wakeup_interrupt(dwc);
3179                 break;
3180         case DWC3_DEVICE_EVENT_HIBER_REQ:
3181                 if (dev_WARN_ONCE(dwc->dev, !dwc->has_hibernation,
3182                                         "unexpected hibernation event\n"))
3183                         break;
3184
3185                 dwc3_gadget_hibernation_interrupt(dwc, event->event_info);
3186                 break;
3187         case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
3188                 dwc3_gadget_linksts_change_interrupt(dwc, event->event_info);
3189                 break;
3190         case DWC3_DEVICE_EVENT_EOPF:
3191                 /* It changed to be suspend event for version 2.30a and above */
3192                 if (dwc->revision >= DWC3_REVISION_230A) {
3193                         /*
3194                          * Ignore suspend event until the gadget enters into
3195                          * USB_STATE_CONFIGURED state.
3196                          */
3197                         if (dwc->gadget.state >= USB_STATE_CONFIGURED)
3198                                 dwc3_gadget_suspend_interrupt(dwc,
3199                                                 event->event_info);
3200                 }
3201                 break;
3202         case DWC3_DEVICE_EVENT_SOF:
3203         case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
3204         case DWC3_DEVICE_EVENT_CMD_CMPL:
3205         case DWC3_DEVICE_EVENT_OVERFLOW:
3206                 break;
3207         default:
3208                 dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type);
3209         }
3210 }
3211
3212 static void dwc3_process_event_entry(struct dwc3 *dwc,
3213                 const union dwc3_event *event)
3214 {
3215         trace_dwc3_event(event->raw, dwc);
3216
3217         if (!event->type.is_devspec)
3218                 dwc3_endpoint_interrupt(dwc, &event->depevt);
3219         else if (event->type.type == DWC3_EVENT_TYPE_DEV)
3220                 dwc3_gadget_interrupt(dwc, &event->devt);
3221         else
3222                 dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw);
3223 }
3224
3225 static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt)
3226 {
3227         struct dwc3 *dwc = evt->dwc;
3228         irqreturn_t ret = IRQ_NONE;
3229         int left;
3230         u32 reg;
3231
3232         left = evt->count;
3233
3234         if (!(evt->flags & DWC3_EVENT_PENDING))
3235                 return IRQ_NONE;
3236
3237         while (left > 0) {
3238                 union dwc3_event event;
3239
3240                 event.raw = *(u32 *) (evt->cache + evt->lpos);
3241
3242                 dwc3_process_event_entry(dwc, &event);
3243
3244                 /*
3245                  * FIXME we wrap around correctly to the next entry as
3246                  * almost all entries are 4 bytes in size. There is one
3247                  * entry which has 12 bytes which is a regular entry
3248                  * followed by 8 bytes data. ATM I don't know how
3249                  * things are organized if we get next to the a
3250                  * boundary so I worry about that once we try to handle
3251                  * that.
3252                  */
3253                 evt->lpos = (evt->lpos + 4) % evt->length;
3254                 left -= 4;
3255         }
3256
3257         evt->count = 0;
3258         evt->flags &= ~DWC3_EVENT_PENDING;
3259         ret = IRQ_HANDLED;
3260
3261         /* Unmask interrupt */
3262         reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0));
3263         reg &= ~DWC3_GEVNTSIZ_INTMASK;
3264         dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
3265
3266         if (dwc->imod_interval) {
3267                 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
3268                 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
3269         }
3270
3271         return ret;
3272 }
3273
3274 static irqreturn_t dwc3_thread_interrupt(int irq, void *_evt)
3275 {
3276         struct dwc3_event_buffer *evt = _evt;
3277         struct dwc3 *dwc = evt->dwc;
3278         unsigned long flags;
3279         irqreturn_t ret = IRQ_NONE;
3280
3281         spin_lock_irqsave(&dwc->lock, flags);
3282         ret = dwc3_process_event_buf(evt);
3283         spin_unlock_irqrestore(&dwc->lock, flags);
3284
3285         return ret;
3286 }
3287
3288 static irqreturn_t dwc3_check_event_buf(struct dwc3_event_buffer *evt)
3289 {
3290         struct dwc3 *dwc = evt->dwc;
3291         u32 amount;
3292         u32 count;
3293         u32 reg;
3294
3295         if (pm_runtime_suspended(dwc->dev)) {
3296                 pm_runtime_get(dwc->dev);
3297                 disable_irq_nosync(dwc->irq_gadget);
3298                 dwc->pending_events = true;
3299                 return IRQ_HANDLED;
3300         }
3301
3302         /*
3303          * With PCIe legacy interrupt, test shows that top-half irq handler can
3304          * be called again after HW interrupt deassertion. Check if bottom-half
3305          * irq event handler completes before caching new event to prevent
3306          * losing events.
3307          */
3308         if (evt->flags & DWC3_EVENT_PENDING)
3309                 return IRQ_HANDLED;
3310
3311         count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(0));
3312         count &= DWC3_GEVNTCOUNT_MASK;
3313         if (!count)
3314                 return IRQ_NONE;
3315
3316         evt->count = count;
3317         evt->flags |= DWC3_EVENT_PENDING;
3318
3319         /* Mask interrupt */
3320         reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0));
3321         reg |= DWC3_GEVNTSIZ_INTMASK;
3322         dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
3323
3324         amount = min(count, evt->length - evt->lpos);
3325         memcpy(evt->cache + evt->lpos, evt->buf + evt->lpos, amount);
3326
3327         if (amount < count)
3328                 memcpy(evt->cache, evt->buf, count - amount);
3329
3330         dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), count);
3331
3332         return IRQ_WAKE_THREAD;
3333 }
3334
3335 static irqreturn_t dwc3_interrupt(int irq, void *_evt)
3336 {
3337         struct dwc3_event_buffer        *evt = _evt;
3338
3339         return dwc3_check_event_buf(evt);
3340 }
3341
3342 static int dwc3_gadget_get_irq(struct dwc3 *dwc)
3343 {
3344         struct platform_device *dwc3_pdev = to_platform_device(dwc->dev);
3345         int irq;
3346
3347         irq = platform_get_irq_byname_optional(dwc3_pdev, "peripheral");
3348         if (irq > 0)
3349                 goto out;
3350
3351         if (irq == -EPROBE_DEFER)
3352                 goto out;
3353
3354         irq = platform_get_irq_byname_optional(dwc3_pdev, "dwc_usb3");
3355         if (irq > 0)
3356                 goto out;
3357
3358         if (irq == -EPROBE_DEFER)
3359                 goto out;
3360
3361         irq = platform_get_irq(dwc3_pdev, 0);
3362         if (irq > 0)
3363                 goto out;
3364
3365         if (!irq)
3366                 irq = -EINVAL;
3367
3368 out:
3369         return irq;
3370 }
3371
3372 /**
3373  * dwc3_gadget_init - initializes gadget related registers
3374  * @dwc: pointer to our controller context structure
3375  *
3376  * Returns 0 on success otherwise negative errno.
3377  */
3378 int dwc3_gadget_init(struct dwc3 *dwc)
3379 {
3380         int ret;
3381         int irq;
3382
3383         irq = dwc3_gadget_get_irq(dwc);
3384         if (irq < 0) {
3385                 ret = irq;
3386                 goto err0;
3387         }
3388
3389         dwc->irq_gadget = irq;
3390
3391         dwc->ep0_trb = dma_alloc_coherent(dwc->sysdev,
3392                                           sizeof(*dwc->ep0_trb) * 2,
3393                                           &dwc->ep0_trb_addr, GFP_KERNEL);
3394         if (!dwc->ep0_trb) {
3395                 dev_err(dwc->dev, "failed to allocate ep0 trb\n");
3396                 ret = -ENOMEM;
3397                 goto err0;
3398         }
3399
3400         dwc->setup_buf = kzalloc(DWC3_EP0_SETUP_SIZE, GFP_KERNEL);
3401         if (!dwc->setup_buf) {
3402                 ret = -ENOMEM;
3403                 goto err1;
3404         }
3405
3406         dwc->bounce = dma_alloc_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE,
3407                         &dwc->bounce_addr, GFP_KERNEL);
3408         if (!dwc->bounce) {
3409                 ret = -ENOMEM;
3410                 goto err2;
3411         }
3412
3413         init_completion(&dwc->ep0_in_setup);
3414
3415         dwc->gadget.ops                 = &dwc3_gadget_ops;
3416         dwc->gadget.speed               = USB_SPEED_UNKNOWN;
3417         dwc->gadget.sg_supported        = true;
3418         dwc->gadget.name                = "dwc3-gadget";
3419         dwc->gadget.lpm_capable         = true;
3420
3421         /*
3422          * FIXME We might be setting max_speed to <SUPER, however versions
3423          * <2.20a of dwc3 have an issue with metastability (documented
3424          * elsewhere in this driver) which tells us we can't set max speed to
3425          * anything lower than SUPER.
3426          *
3427          * Because gadget.max_speed is only used by composite.c and function
3428          * drivers (i.e. it won't go into dwc3's registers) we are allowing this
3429          * to happen so we avoid sending SuperSpeed Capability descriptor
3430          * together with our BOS descriptor as that could confuse host into
3431          * thinking we can handle super speed.
3432          *
3433          * Note that, in fact, we won't even support GetBOS requests when speed
3434          * is less than super speed because we don't have means, yet, to tell
3435          * composite.c that we are USB 2.0 + LPM ECN.
3436          */
3437         if (dwc->revision < DWC3_REVISION_220A &&
3438             !dwc->dis_metastability_quirk)
3439                 dev_info(dwc->dev, "changing max_speed on rev %08x\n",
3440                                 dwc->revision);
3441
3442         dwc->gadget.max_speed           = dwc->maximum_speed;
3443
3444         /*
3445          * REVISIT: Here we should clear all pending IRQs to be
3446          * sure we're starting from a well known location.
3447          */
3448
3449         ret = dwc3_gadget_init_endpoints(dwc, dwc->num_eps);
3450         if (ret)
3451                 goto err3;
3452
3453         ret = usb_add_gadget_udc(dwc->dev, &dwc->gadget);
3454         if (ret) {
3455                 dev_err(dwc->dev, "failed to register udc\n");
3456                 goto err4;
3457         }
3458
3459         dwc3_gadget_set_speed(&dwc->gadget, dwc->maximum_speed);
3460
3461         return 0;
3462
3463 err4:
3464         dwc3_gadget_free_endpoints(dwc);
3465
3466 err3:
3467         dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
3468                         dwc->bounce_addr);
3469
3470 err2:
3471         kfree(dwc->setup_buf);
3472
3473 err1:
3474         dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
3475                         dwc->ep0_trb, dwc->ep0_trb_addr);
3476
3477 err0:
3478         return ret;
3479 }
3480
3481 /* -------------------------------------------------------------------------- */
3482
3483 void dwc3_gadget_exit(struct dwc3 *dwc)
3484 {
3485         usb_del_gadget_udc(&dwc->gadget);
3486         dwc3_gadget_free_endpoints(dwc);
3487         dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
3488                           dwc->bounce_addr);
3489         kfree(dwc->setup_buf);
3490         dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
3491                           dwc->ep0_trb, dwc->ep0_trb_addr);
3492 }
3493
3494 int dwc3_gadget_suspend(struct dwc3 *dwc)
3495 {
3496         if (!dwc->gadget_driver)
3497                 return 0;
3498
3499         dwc3_gadget_run_stop(dwc, false, false);
3500         dwc3_disconnect_gadget(dwc);
3501         __dwc3_gadget_stop(dwc);
3502
3503         return 0;
3504 }
3505
3506 int dwc3_gadget_resume(struct dwc3 *dwc)
3507 {
3508         int                     ret;
3509
3510         if (!dwc->gadget_driver)
3511                 return 0;
3512
3513         ret = __dwc3_gadget_start(dwc);
3514         if (ret < 0)
3515                 goto err0;
3516
3517         ret = dwc3_gadget_run_stop(dwc, true, false);
3518         if (ret < 0)
3519                 goto err1;
3520
3521         return 0;
3522
3523 err1:
3524         __dwc3_gadget_stop(dwc);
3525
3526 err0:
3527         return ret;
3528 }
3529
3530 void dwc3_gadget_process_pending_events(struct dwc3 *dwc)
3531 {
3532         if (dwc->pending_events) {
3533                 dwc3_interrupt(dwc->irq_gadget, dwc->ev_buf);
3534                 dwc->pending_events = false;
3535                 enable_irq(dwc->irq_gadget);
3536         }
3537 }