drivers/usb/dwc3/gadget.c

   1 /**
   2  * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
   3  *
   4  * Copyright (C) 2010-2011 Texas Instruments Incorporated - http://www.ti.com
   5  *
   6  * Authors: Felipe Balbi <balbi@ti.com>,
   7  *          Sebastian Andrzej Siewior <bigeasy@linutronix.de>
   8  *
   9  * This program is free software: you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License version 2  of
  11  * the License as published by the Free Software Foundation.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  */
  18
  19 #include <linux/kernel.h>
  20 #include <linux/delay.h>
  21 #include <linux/slab.h>
  22 #include <linux/spinlock.h>
  23 #include <linux/platform_device.h>
  24 #include <linux/pm_runtime.h>
  25 #include <linux/interrupt.h>
  26 #include <linux/io.h>
  27 #include <linux/list.h>
  28 #include <linux/dma-mapping.h>
  29
  30 #include <linux/usb/ch9.h>
  31 #include <linux/usb/gadget.h>
  32
  33 #include "core.h"
  34 #include "gadget.h"
  35 #include "io.h"
  36
  37 /**
  38  * dwc3_gadget_set_test_mode - Enables USB2 Test Modes
  39  * @dwc: pointer to our context structure
  40  * @mode: the mode to set (J, K SE0 NAK, Force Enable)
  41  *
  42  * Caller should take care of locking. This function will
  43  * return 0 on success or -EINVAL if wrong Test Selector
  44  * is passed
  45  */
  46 int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode)
  47 {
  48         u32             reg;
  49
  50         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
  51         reg &= ~DWC3_DCTL_TSTCTRL_MASK;
  52
  53         switch (mode) {
  54         case TEST_J:
  55         case TEST_K:
  56         case TEST_SE0_NAK:
  57         case TEST_PACKET:
  58         case TEST_FORCE_EN:
  59                 reg |= mode << 1;
  60                 break;
  61         default:
  62                 return -EINVAL;
  63         }
  64
  65         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
  66
  67         return 0;
  68 }
  69
  70 /**
  71  * dwc3_gadget_set_link_state - Sets USB Link to a particular State
  72  * @dwc: pointer to our context structure
  73  * @state: the state to put link into
  74  *
  75  * Caller should take care of locking. This function will
  76  * return 0 on success or -ETIMEDOUT.
  77  */
  78 int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state)
  79 {
  80         int             retries = 10000;
  81         u32             reg;
  82
  83         /*
  84          * Wait until device controller is ready. Only applies to 1.94a and
  85          * later RTL.
  86          */
  87         if (dwc->revision >= DWC3_REVISION_194A) {
  88                 while (--retries) {
  89                         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
  90                         if (reg & DWC3_DSTS_DCNRD)
  91                                 udelay(5);
  92                         else
  93                                 break;
  94                 }
  95
  96                 if (retries <= 0)
  97                         return -ETIMEDOUT;
  98         }
  99
 100         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
 101         reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
 102
 103         /* set requested state */
 104         reg |= DWC3_DCTL_ULSTCHNGREQ(state);
 105         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
 106
 107         /*
 108          * The following code is racy when called from dwc3_gadget_wakeup,
 109          * and is not needed, at least on newer versions
 110          */
 111         if (dwc->revision >= DWC3_REVISION_194A)
 112                 return 0;
 113
 114         /* wait for a change in DSTS */
 115         retries = 10000;
 116         while (--retries) {
 117                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
 118
 119                 if (DWC3_DSTS_USBLNKST(reg) == state)
 120                         return 0;
 121
 122                 udelay(5);
 123         }
 124
 125         dev_vdbg(dwc->dev, "link state change request timed out\n");
 126
 127         return -ETIMEDOUT;
 128 }
 129
 130 /**
 131  * dwc3_gadget_resize_tx_fifos - reallocate fifo spaces for current use-case
 132  * @dwc: pointer to our context structure
 133  *
 134  * This function will a best effort FIFO allocation in order
 135  * to improve FIFO usage and throughput, while still allowing
 136  * us to enable as many endpoints as possible.
 137  *
 138  * Keep in mind that this operation will be highly dependent
 139  * on the configured size for RAM1 - which contains TxFifo -,
 140  * the amount of endpoints enabled on coreConsultant tool, and
 141  * the width of the Master Bus.
 142  *
 143  * In the ideal world, we would always be able to satisfy the
 144  * following equation:
 145  *
 146  * ((512 + 2 * MDWIDTH-Bytes) + (Number of IN Endpoints - 1) * \
 147  * (3 * (1024 + MDWIDTH-Bytes) + MDWIDTH-Bytes)) / MDWIDTH-Bytes
 148  *
 149  * Unfortunately, due to many variables that's not always the case.
 150  */
 151 int dwc3_gadget_resize_tx_fifos(struct dwc3 *dwc)
 152 {
 153         int             last_fifo_depth = 0;
 154         int             ram1_depth;
 155         int             fifo_size;
 156         int             mdwidth;
 157         int             num;
 158
 159         if (!dwc->needs_fifo_resize)
 160                 return 0;
 161
 162         ram1_depth = DWC3_RAM1_DEPTH(dwc->hwparams.hwparams7);
 163         mdwidth = DWC3_MDWIDTH(dwc->hwparams.hwparams0);
 164
 165         /* MDWIDTH is represented in bits, we need it in bytes */
 166         mdwidth >>= 3;
 167
 168         /*
 169          * FIXME For now we will only allocate 1 wMaxPacketSize space
 170          * for each enabled endpoint, later patches will come to
 171          * improve this algorithm so that we better use the internal
 172          * FIFO space
 173          */
 174         for (num = 0; num < DWC3_ENDPOINTS_NUM; num++) {
 175                 struct dwc3_ep  *dep = dwc->eps[num];
 176                 int             fifo_number = dep->number >> 1;
 177                 int             mult = 1;
 178                 int             tmp;
 179
 180                 if (!(dep->number & 1))
 181                         continue;
 182
 183                 if (!(dep->flags & DWC3_EP_ENABLED))
 184                         continue;
 185
 186                 if (usb_endpoint_xfer_bulk(dep->endpoint.desc)
 187                                 || usb_endpoint_xfer_isoc(dep->endpoint.desc))
 188                         mult = 3;
 189
 190                 /*
 191                  * REVISIT: the following assumes we will always have enough
 192                  * space available on the FIFO RAM for all possible use cases.
 193                  * Make sure that's true somehow and change FIFO allocation
 194                  * accordingly.
 195                  *
 196                  * If we have Bulk or Isochronous endpoints, we want
 197                  * them to be able to be very, very fast. So we're giving
 198                  * those endpoints a fifo_size which is enough for 3 full
 199                  * packets
 200                  */
 201                 tmp = mult * (dep->endpoint.maxpacket + mdwidth);
 202                 tmp += mdwidth;
 203
 204                 fifo_size = DIV_ROUND_UP(tmp, mdwidth);
 205
 206                 fifo_size |= (last_fifo_depth << 16);
 207
 208                 dev_vdbg(dwc->dev, "%s: Fifo Addr %04x Size %d\n",
 209                                 dep->name, last_fifo_depth, fifo_size & 0xffff);
 210
 211                 dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(fifo_number),
 212                                 fifo_size);
 213
 214                 last_fifo_depth += (fifo_size & 0xffff);
 215         }
 216
 217         return 0;
 218 }
 219
 220 void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
 221                 int status)
 222 {
 223         struct dwc3                     *dwc = dep->dwc;
 224         int                             i;
 225
 226         if (req->queued) {
 227                 i = 0;
 228                 do {
 229                         dep->busy_slot++;
 230                         /*
 231                          * Skip LINK TRB. We can't use req->trb and check for
 232                          * DWC3_TRBCTL_LINK_TRB because it points the TRB we
 233                          * just completed (not the LINK TRB).
 234                          */
 235                         if (((dep->busy_slot & DWC3_TRB_MASK) ==
 236                                 DWC3_TRB_NUM- 1) &&
 237                                 usb_endpoint_xfer_isoc(dep->endpoint.desc))
 238                                 dep->busy_slot++;
 239                 } while(++i < req->request.num_mapped_sgs);
 240                 req->queued = false;
 241         }
 242         list_del(&req->list);
 243         req->trb = NULL;
 244
 245         if (req->request.status == -EINPROGRESS)
 246                 req->request.status = status;
 247
 248         if (dwc->ep0_bounced && dep->number == 0)
 249                 dwc->ep0_bounced = false;
 250         else
 251                 usb_gadget_unmap_request(&dwc->gadget, &req->request,
 252                                 req->direction);
 253
 254         dev_dbg(dwc->dev, "request %p from %s completed %d/%d ===> %d\n",
 255                         req, dep->name, req->request.actual,
 256                         req->request.length, status);
 257
 258         spin_unlock(&dwc->lock);
 259         req->request.complete(&dep->endpoint, &req->request);
 260         spin_lock(&dwc->lock);
 261 }
 262
 263 static const char *dwc3_gadget_ep_cmd_string(u8 cmd)
 264 {
 265         switch (cmd) {
 266         case DWC3_DEPCMD_DEPSTARTCFG:
 267                 return "Start New Configuration";
 268         case DWC3_DEPCMD_ENDTRANSFER:
 269                 return "End Transfer";
 270         case DWC3_DEPCMD_UPDATETRANSFER:
 271                 return "Update Transfer";
 272         case DWC3_DEPCMD_STARTTRANSFER:
 273                 return "Start Transfer";
 274         case DWC3_DEPCMD_CLEARSTALL:
 275                 return "Clear Stall";
 276         case DWC3_DEPCMD_SETSTALL:
 277                 return "Set Stall";
 278         case DWC3_DEPCMD_GETEPSTATE:
 279                 return "Get Endpoint State";
 280         case DWC3_DEPCMD_SETTRANSFRESOURCE:
 281                 return "Set Endpoint Transfer Resource";
 282         case DWC3_DEPCMD_SETEPCONFIG:
 283                 return "Set Endpoint Configuration";
 284         default:
 285                 return "UNKNOWN command";
 286         }
 287 }
 288
 289 int dwc3_send_gadget_generic_command(struct dwc3 *dwc, int cmd, u32 param)
 290 {
 291         u32             timeout = 500;
 292         u32             reg;
 293
 294         dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param);
 295         dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT);
 296
 297         do {
 298                 reg = dwc3_readl(dwc->regs, DWC3_DGCMD);
 299                 if (!(reg & DWC3_DGCMD_CMDACT)) {
 300                         dev_vdbg(dwc->dev, "Command Complete --> %d\n",
 301                                         DWC3_DGCMD_STATUS(reg));
 302                         return 0;
 303                 }
 304
 305                 /*
 306                  * We can't sleep here, because it's also called from
 307                  * interrupt context.
 308                  */
 309                 timeout--;
 310                 if (!timeout)
 311                         return -ETIMEDOUT;
 312                 udelay(1);
 313         } while (1);
 314 }
 315
 316 int dwc3_send_gadget_ep_cmd(struct dwc3 *dwc, unsigned ep,
 317                 unsigned cmd, struct dwc3_gadget_ep_cmd_params *params)
 318 {
 319         struct dwc3_ep          *dep = dwc->eps[ep];
 320         u32                     timeout = 500;
 321         u32                     reg;
 322
 323         dev_vdbg(dwc->dev, "%s: cmd '%s' params %08x %08x %08x\n",
 324                         dep->name,
 325                         dwc3_gadget_ep_cmd_string(cmd), params->param0,
 326                         params->param1, params->param2);
 327
 328         dwc3_writel(dwc->regs, DWC3_DEPCMDPAR0(ep), params->param0);
 329         dwc3_writel(dwc->regs, DWC3_DEPCMDPAR1(ep), params->param1);
 330         dwc3_writel(dwc->regs, DWC3_DEPCMDPAR2(ep), params->param2);
 331
 332         dwc3_writel(dwc->regs, DWC3_DEPCMD(ep), cmd | DWC3_DEPCMD_CMDACT);
 333         do {
 334                 reg = dwc3_readl(dwc->regs, DWC3_DEPCMD(ep));
 335                 if (!(reg & DWC3_DEPCMD_CMDACT)) {
 336                         dev_vdbg(dwc->dev, "Command Complete --> %d\n",
 337                                         DWC3_DEPCMD_STATUS(reg));
 338                         return 0;
 339                 }
 340
 341                 /*
 342                  * We can't sleep here, because it is also called from
 343                  * interrupt context.
 344                  */
 345                 timeout--;
 346                 if (!timeout)
 347                         return -ETIMEDOUT;
 348
 349                 udelay(1);
 350         } while (1);
 351 }
 352
 353 static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep,
 354                 struct dwc3_trb *trb)
 355 {
 356         u32             offset = (char *) trb - (char *) dep->trb_pool;
 357
 358         return dep->trb_pool_dma + offset;
 359 }
 360
 361 static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)
 362 {
 363         struct dwc3             *dwc = dep->dwc;
 364
 365         if (dep->trb_pool)
 366                 return 0;
 367
 368         if (dep->number == 0 || dep->number == 1)
 369                 return 0;
 370
 371         dep->trb_pool = dma_alloc_coherent(dwc->dev,
 372                         sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
 373                         &dep->trb_pool_dma, GFP_KERNEL);
 374         if (!dep->trb_pool) {
 375                 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",
 376                                 dep->name);
 377                 return -ENOMEM;
 378         }
 379
 380         return 0;
 381 }
 382
 383 static void dwc3_free_trb_pool(struct dwc3_ep *dep)
 384 {
 385         struct dwc3             *dwc = dep->dwc;
 386
 387         dma_free_coherent(dwc->dev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
 388                         dep->trb_pool, dep->trb_pool_dma);
 389
 390         dep->trb_pool = NULL;
 391         dep->trb_pool_dma = 0;
 392 }
 393
 394 static int dwc3_gadget_start_config(struct dwc3 *dwc, struct dwc3_ep *dep)
 395 {
 396         struct dwc3_gadget_ep_cmd_params params;
 397         u32                     cmd;
 398
 399         memset(&params, 0x00, sizeof(params));
 400
 401         if (dep->number != 1) {
 402                 cmd = DWC3_DEPCMD_DEPSTARTCFG;
 403                 /* XferRscIdx == 0 for ep0 and 2 for the remaining */
 404                 if (dep->number > 1) {
 405                         if (dwc->start_config_issued)
 406                                 return 0;
 407                         dwc->start_config_issued = true;
 408                         cmd |= DWC3_DEPCMD_PARAM(2);
 409                 }
 410
 411                 return dwc3_send_gadget_ep_cmd(dwc, 0, cmd, &params);
 412         }
 413
 414         return 0;
 415 }
 416
 417 static int dwc3_gadget_set_ep_config(struct dwc3 *dwc, struct dwc3_ep *dep,
 418                 const struct usb_endpoint_descriptor *desc,
 419                 const struct usb_ss_ep_comp_descriptor *comp_desc,
 420                 bool ignore)
 421 {
 422         struct dwc3_gadget_ep_cmd_params params;
 423
 424         memset(&params, 0x00, sizeof(params));
 425
 426         params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc))
 427                 | DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc));
 428
 429         /* Burst size is only needed in SuperSpeed mode */
 430         if (dwc->gadget.speed == USB_SPEED_SUPER) {
 431                 u32 burst = dep->endpoint.maxburst - 1;
 432
 433                 params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst);
 434         }
 435
 436         if (ignore)
 437                 params.param0 |= DWC3_DEPCFG_IGN_SEQ_NUM;
 438
 439         params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN
 440                 | DWC3_DEPCFG_XFER_NOT_READY_EN;
 441
 442         if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
 443                 params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE
 444                         | DWC3_DEPCFG_STREAM_EVENT_EN;
 445                 dep->stream_capable = true;
 446         }
 447
 448         if (usb_endpoint_xfer_isoc(desc))
 449                 params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN;
 450
 451         /*
 452          * We are doing 1:1 mapping for endpoints, meaning
 453          * Physical Endpoints 2 maps to Logical Endpoint 2 and
 454          * so on. We consider the direction bit as part of the physical
 455          * endpoint number. So USB endpoint 0x81 is 0x03.
 456          */
 457         params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number);
 458
 459         /*
 460          * We must use the lower 16 TX FIFOs even though
 461          * HW might have more
 462          */
 463         if (dep->direction)
 464                 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1);
 465
 466         if (desc->bInterval) {
 467                 params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(desc->bInterval - 1);
 468                 dep->interval = 1 << (desc->bInterval - 1);
 469         }
 470
 471         return dwc3_send_gadget_ep_cmd(dwc, dep->number,
 472                         DWC3_DEPCMD_SETEPCONFIG, &params);
 473 }
 474
 475 static int dwc3_gadget_set_xfer_resource(struct dwc3 *dwc, struct dwc3_ep *dep)
 476 {
 477         struct dwc3_gadget_ep_cmd_params params;
 478
 479         memset(&params, 0x00, sizeof(params));
 480
 481         params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1);
 482
 483         return dwc3_send_gadget_ep_cmd(dwc, dep->number,
 484                         DWC3_DEPCMD_SETTRANSFRESOURCE, &params);
 485 }
 486
 487 /**
 488  * __dwc3_gadget_ep_enable - Initializes a HW endpoint
 489  * @dep: endpoint to be initialized
 490  * @desc: USB Endpoint Descriptor
 491  *
 492  * Caller should take care of locking
 493  */
 494 static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep,
 495                 const struct usb_endpoint_descriptor *desc,
 496                 const struct usb_ss_ep_comp_descriptor *comp_desc,
 497                 bool ignore)
 498 {
 499         struct dwc3             *dwc = dep->dwc;
 500         u32                     reg;
 501         int                     ret = -ENOMEM;
 502
 503         dev_vdbg(dwc->dev, "Enabling %s\n", dep->name);
 504
 505         if (!(dep->flags & DWC3_EP_ENABLED)) {
 506                 ret = dwc3_gadget_start_config(dwc, dep);
 507                 if (ret)
 508                         return ret;
 509         }
 510
 511         ret = dwc3_gadget_set_ep_config(dwc, dep, desc, comp_desc, ignore);
 512         if (ret)
 513                 return ret;
 514
 515         if (!(dep->flags & DWC3_EP_ENABLED)) {
 516                 struct dwc3_trb *trb_st_hw;
 517                 struct dwc3_trb *trb_link;
 518
 519                 ret = dwc3_gadget_set_xfer_resource(dwc, dep);
 520                 if (ret)
 521                         return ret;
 522
 523                 dep->endpoint.desc = desc;
 524                 dep->comp_desc = comp_desc;
 525                 dep->type = usb_endpoint_type(desc);
 526                 dep->flags |= DWC3_EP_ENABLED;
 527
 528                 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
 529                 reg |= DWC3_DALEPENA_EP(dep->number);
 530                 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
 531
 532                 if (!usb_endpoint_xfer_isoc(desc))
 533                         return 0;
 534
 535                 /* Link TRB for ISOC. The HWO bit is never reset */
 536                 trb_st_hw = &dep->trb_pool[0];
 537
 538                 trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1];
 539                 memset(trb_link, 0, sizeof(*trb_link));
 540
 541                 trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
 542                 trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
 543                 trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB;
 544                 trb_link->ctrl |= DWC3_TRB_CTRL_HWO;
 545         }
 546
 547         return 0;
 548 }
 549
 550 static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum);
 551 static void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep)
 552 {
 553         struct dwc3_request             *req;
 554
 555         if (!list_empty(&dep->req_queued)) {
 556                 dwc3_stop_active_transfer(dwc, dep->number);
 557
 558                 /* - giveback all requests to gadget driver */
 559                 while (!list_empty(&dep->req_queued)) {
 560                         req = next_request(&dep->req_queued);
 561
 562                         dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
 563                 }
 564         }
 565
 566         while (!list_empty(&dep->request_list)) {
 567                 req = next_request(&dep->request_list);
 568
 569                 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
 570         }
 571 }
 572
 573 /**
 574  * __dwc3_gadget_ep_disable - Disables a HW endpoint
 575  * @dep: the endpoint to disable
 576  *
 577  * This function also removes requests which are currently processed ny the
 578  * hardware and those which are not yet scheduled.
 579  * Caller should take care of locking.
 580  */
 581 static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
 582 {
 583         struct dwc3             *dwc = dep->dwc;
 584         u32                     reg;
 585
 586         dwc3_remove_requests(dwc, dep);
 587
 588         /* make sure HW endpoint isn't stalled */
 589         if (dep->flags & DWC3_EP_STALL)
 590                 __dwc3_gadget_ep_set_halt(dep, 0, false);
 591
 592         reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
 593         reg &= ~DWC3_DALEPENA_EP(dep->number);
 594         dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
 595
 596         dep->stream_capable = false;
 597         dep->endpoint.desc = NULL;
 598         dep->comp_desc = NULL;
 599         dep->type = 0;
 600         dep->flags = 0;
 601
 602         return 0;
 603 }
 604
 605 /* -------------------------------------------------------------------------- */
 606
 607 static int dwc3_gadget_ep0_enable(struct usb_ep *ep,
 608                 const struct usb_endpoint_descriptor *desc)
 609 {
 610         return -EINVAL;
 611 }
 612
 613 static int dwc3_gadget_ep0_disable(struct usb_ep *ep)
 614 {
 615         return -EINVAL;
 616 }
 617
 618 /* -------------------------------------------------------------------------- */
 619
 620 static int dwc3_gadget_ep_enable(struct usb_ep *ep,
 621                 const struct usb_endpoint_descriptor *desc)
 622 {
 623         struct dwc3_ep                  *dep;
 624         struct dwc3                     *dwc;
 625         unsigned long                   flags;
 626         int                             ret;
 627
 628         if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
 629                 pr_debug("dwc3: invalid parameters\n");
 630                 return -EINVAL;
 631         }
 632
 633         if (!desc->wMaxPacketSize) {
 634                 pr_debug("dwc3: missing wMaxPacketSize\n");
 635                 return -EINVAL;
 636         }
 637
 638         dep = to_dwc3_ep(ep);
 639         dwc = dep->dwc;
 640
 641         if (dep->flags & DWC3_EP_ENABLED) {
 642                 dev_WARN_ONCE(dwc->dev, true, "%s is already enabled\n",
 643                                 dep->name);
 644                 return 0;
 645         }
 646
 647         switch (usb_endpoint_type(desc)) {
 648         case USB_ENDPOINT_XFER_CONTROL:
 649                 strlcat(dep->name, "-control", sizeof(dep->name));
 650                 break;
 651         case USB_ENDPOINT_XFER_ISOC:
 652                 strlcat(dep->name, "-isoc", sizeof(dep->name));
 653                 break;
 654         case USB_ENDPOINT_XFER_BULK:
 655                 strlcat(dep->name, "-bulk", sizeof(dep->name));
 656                 break;
 657         case USB_ENDPOINT_XFER_INT:
 658                 strlcat(dep->name, "-int", sizeof(dep->name));
 659                 break;
 660         default:
 661                 dev_err(dwc->dev, "invalid endpoint transfer type\n");
 662         }
 663
 664         spin_lock_irqsave(&dwc->lock, flags);
 665         ret = __dwc3_gadget_ep_enable(dep, desc, ep->comp_desc, false);
 666         spin_unlock_irqrestore(&dwc->lock, flags);
 667
 668         return ret;
 669 }
 670
 671 static int dwc3_gadget_ep_disable(struct usb_ep *ep)
 672 {
 673         struct dwc3_ep                  *dep;
 674         struct dwc3                     *dwc;
 675         unsigned long                   flags;
 676         int                             ret;
 677
 678         if (!ep) {
 679                 pr_debug("dwc3: invalid parameters\n");
 680                 return -EINVAL;
 681         }
 682
 683         dep = to_dwc3_ep(ep);
 684         dwc = dep->dwc;
 685
 686         if (!(dep->flags & DWC3_EP_ENABLED)) {
 687                 dev_WARN_ONCE(dwc->dev, true, "%s is already disabled\n",
 688                                 dep->name);
 689                 return 0;
 690         }
 691
 692         snprintf(dep->name, sizeof(dep->name), "ep%d%s",
 693                         dep->number >> 1,
 694                         (dep->number & 1) ? "in" : "out");
 695
 696         spin_lock_irqsave(&dwc->lock, flags);
 697         ret = __dwc3_gadget_ep_disable(dep);
 698         spin_unlock_irqrestore(&dwc->lock, flags);
 699
 700         return ret;
 701 }
 702
 703 static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep,
 704         gfp_t gfp_flags)
 705 {
 706         struct dwc3_request             *req;
 707         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
 708         struct dwc3                     *dwc = dep->dwc;
 709
 710         req = kzalloc(sizeof(*req), gfp_flags);
 711         if (!req) {
 712                 dev_err(dwc->dev, "not enough memory\n");
 713                 return NULL;
 714         }
 715
 716         req->epnum      = dep->number;
 717         req->dep        = dep;
 718
 719         return &req->request;
 720 }
 721
 722 static void dwc3_gadget_ep_free_request(struct usb_ep *ep,
 723                 struct usb_request *request)
 724 {
 725         struct dwc3_request             *req = to_dwc3_request(request);
 726
 727         kfree(req);
 728 }
 729
 730 /**
 731  * dwc3_prepare_one_trb - setup one TRB from one request
 732  * @dep: endpoint for which this request is prepared
 733  * @req: dwc3_request pointer
 734  */
 735 static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
 736                 struct dwc3_request *req, dma_addr_t dma,
 737                 unsigned length, unsigned last, unsigned chain, unsigned node)
 738 {
 739         struct dwc3             *dwc = dep->dwc;
 740         struct dwc3_trb         *trb;
 741
 742         dev_vdbg(dwc->dev, "%s: req %p dma %08llx length %d%s%s\n",
 743                         dep->name, req, (unsigned long long) dma,
 744                         length, last ? " last" : "",
 745                         chain ? " chain" : "");
 746
 747         /* Skip the LINK-TRB on ISOC */
 748         if (((dep->free_slot & DWC3_TRB_MASK) == DWC3_TRB_NUM - 1) &&
 749                         usb_endpoint_xfer_isoc(dep->endpoint.desc))
 750                 dep->free_slot++;
 751
 752         trb = &dep->trb_pool[dep->free_slot & DWC3_TRB_MASK];
 753
 754         if (!req->trb) {
 755                 dwc3_gadget_move_request_queued(req);
 756                 req->trb = trb;
 757                 req->trb_dma = dwc3_trb_dma_offset(dep, trb);
 758                 req->start_slot = dep->free_slot & DWC3_TRB_MASK;
 759         }
 760
 761         dep->free_slot++;
 762
 763         trb->size = DWC3_TRB_SIZE_LENGTH(length);
 764         trb->bpl = lower_32_bits(dma);
 765         trb->bph = upper_32_bits(dma);
 766
 767         switch (usb_endpoint_type(dep->endpoint.desc)) {
 768         case USB_ENDPOINT_XFER_CONTROL:
 769                 trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP;
 770                 break;
 771
 772         case USB_ENDPOINT_XFER_ISOC:
 773                 if (!node)
 774                         trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;
 775                 else
 776                         trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS;
 777
 778                 if (!req->request.no_interrupt && !chain)
 779                         trb->ctrl |= DWC3_TRB_CTRL_IOC;
 780                 break;
 781
 782         case USB_ENDPOINT_XFER_BULK:
 783         case USB_ENDPOINT_XFER_INT:
 784                 trb->ctrl = DWC3_TRBCTL_NORMAL;
 785                 break;
 786         default:
 787                 /*
 788                  * This is only possible with faulty memory because we
 789                  * checked it already :)
 790                  */
 791                 BUG();
 792         }
 793
 794         if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
 795                 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
 796                 trb->ctrl |= DWC3_TRB_CTRL_CSP;
 797         } else if (last) {
 798                 trb->ctrl |= DWC3_TRB_CTRL_LST;
 799         }
 800
 801         if (chain)
 802                 trb->ctrl |= DWC3_TRB_CTRL_CHN;
 803
 804         if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable)
 805                 trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(req->request.stream_id);
 806
 807         trb->ctrl |= DWC3_TRB_CTRL_HWO;
 808 }
 809
 810 /*
 811  * dwc3_prepare_trbs - setup TRBs from requests
 812  * @dep: endpoint for which requests are being prepared
 813  * @starting: true if the endpoint is idle and no requests are queued.
 814  *
 815  * The function goes through the requests list and sets up TRBs for the
 816  * transfers. The function returns once there are no more TRBs available or
 817  * it runs out of requests.
 818  */
 819 static void dwc3_prepare_trbs(struct dwc3_ep *dep, bool starting)
 820 {
 821         struct dwc3_request     *req, *n;
 822         u32                     trbs_left;
 823         u32                     max;
 824         unsigned int            last_one = 0;
 825
 826         BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM);
 827
 828         /* the first request must not be queued */
 829         trbs_left = (dep->busy_slot - dep->free_slot) & DWC3_TRB_MASK;
 830
 831         /* Can't wrap around on a non-isoc EP since there's no link TRB */
 832         if (!usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
 833                 max = DWC3_TRB_NUM - (dep->free_slot & DWC3_TRB_MASK);
 834                 if (trbs_left > max)
 835                         trbs_left = max;
 836         }
 837
 838         /*
 839          * If busy & slot are equal than it is either full or empty. If we are
 840          * starting to process requests then we are empty. Otherwise we are
 841          * full and don't do anything
 842          */
 843         if (!trbs_left) {
 844                 if (!starting)
 845                         return;
 846                 trbs_left = DWC3_TRB_NUM;
 847                 /*
 848                  * In case we start from scratch, we queue the ISOC requests
 849                  * starting from slot 1. This is done because we use ring
 850                  * buffer and have no LST bit to stop us. Instead, we place
 851                  * IOC bit every TRB_NUM/4. We try to avoid having an interrupt
 852                  * after the first request so we start at slot 1 and have
 853                  * 7 requests proceed before we hit the first IOC.
 854                  * Other transfer types don't use the ring buffer and are
 855                  * processed from the first TRB until the last one. Since we
 856                  * don't wrap around we have to start at the beginning.
 857                  */
 858                 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
 859                         dep->busy_slot = 1;
 860                         dep->free_slot = 1;
 861                 } else {
 862                         dep->busy_slot = 0;
 863                         dep->free_slot = 0;
 864                 }
 865         }
 866
 867         /* The last TRB is a link TRB, not used for xfer */
 868         if ((trbs_left <= 1) && usb_endpoint_xfer_isoc(dep->endpoint.desc))
 869                 return;
 870
 871         list_for_each_entry_safe(req, n, &dep->request_list, list) {
 872                 unsigned        length;
 873                 dma_addr_t      dma;
 874                 last_one = false;
 875
 876                 if (req->request.num_mapped_sgs > 0) {
 877                         struct usb_request *request = &req->request;
 878                         struct scatterlist *sg = request->sg;
 879                         struct scatterlist *s;
 880                         int             i;
 881
 882                         for_each_sg(sg, s, request->num_mapped_sgs, i) {
 883                                 unsigned chain = true;
 884
 885                                 length = sg_dma_len(s);
 886                                 dma = sg_dma_address(s);
 887
 888                                 if (i == (request->num_mapped_sgs - 1) ||
 889                                                 sg_is_last(s)) {
 890                                         if (list_is_last(&req->list,
 891                                                         &dep->request_list))
 892                                                 last_one = true;
 893                                         chain = false;
 894                                 }
 895
 896                                 trbs_left--;
 897                                 if (!trbs_left)
 898                                         last_one = true;
 899
 900                                 if (last_one)
 901                                         chain = false;
 902
 903                                 dwc3_prepare_one_trb(dep, req, dma, length,
 904                                                 last_one, chain, i);
 905
 906                                 if (last_one)
 907                                         break;
 908                         }
 909                 } else {
 910                         dma = req->request.dma;
 911                         length = req->request.length;
 912                         trbs_left--;
 913
 914                         if (!trbs_left)
 915                                 last_one = 1;
 916
 917                         /* Is this the last request? */
 918                         if (list_is_last(&req->list, &dep->request_list))
 919                                 last_one = 1;
 920
 921                         dwc3_prepare_one_trb(dep, req, dma, length,
 922                                         last_one, false, 0);
 923
 924                         if (last_one)
 925                                 break;
 926                 }
 927         }
 928 }
 929
 930 static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep, u16 cmd_param,
 931                 int start_new)
 932 {
 933         struct dwc3_gadget_ep_cmd_params params;
 934         struct dwc3_request             *req;
 935         struct dwc3                     *dwc = dep->dwc;
 936         int                             ret;
 937         u32                             cmd;
 938
 939         if (start_new && (dep->flags & DWC3_EP_BUSY)) {
 940                 dev_vdbg(dwc->dev, "%s: endpoint busy\n", dep->name);
 941                 return -EBUSY;
 942         }
 943         dep->flags &= ~DWC3_EP_PENDING_REQUEST;
 944
 945         /*
 946          * If we are getting here after a short-out-packet we don't enqueue any
 947          * new requests as we try to set the IOC bit only on the last request.
 948          */
 949         if (start_new) {
 950                 if (list_empty(&dep->req_queued))
 951                         dwc3_prepare_trbs(dep, start_new);
 952
 953                 /* req points to the first request which will be sent */
 954                 req = next_request(&dep->req_queued);
 955         } else {
 956                 dwc3_prepare_trbs(dep, start_new);
 957
 958                 /*
 959                  * req points to the first request where HWO changed from 0 to 1
 960                  */
 961                 req = next_request(&dep->req_queued);
 962         }
 963         if (!req) {
 964                 dep->flags |= DWC3_EP_PENDING_REQUEST;
 965                 return 0;
 966         }
 967
 968         memset(&params, 0, sizeof(params));
 969
 970         if (start_new) {
 971                 params.param0 = upper_32_bits(req->trb_dma);
 972                 params.param1 = lower_32_bits(req->trb_dma);
 973                 cmd = DWC3_DEPCMD_STARTTRANSFER;
 974         } else {
 975                 cmd = DWC3_DEPCMD_UPDATETRANSFER;
 976         }
 977
 978         cmd |= DWC3_DEPCMD_PARAM(cmd_param);
 979         ret = dwc3_send_gadget_ep_cmd(dwc, dep->number, cmd, &params);
 980         if (ret < 0) {
 981                 dev_dbg(dwc->dev, "failed to send STARTTRANSFER command\n");
 982
 983                 /*
 984                  * FIXME we need to iterate over the list of requests
 985                  * here and stop, unmap, free and del each of the linked
 986                  * requests instead of what we do now.
 987                  */
 988                 usb_gadget_unmap_request(&dwc->gadget, &req->request,
 989                                 req->direction);
 990                 list_del(&req->list);
 991                 return ret;
 992         }
 993
 994         dep->flags |= DWC3_EP_BUSY;
 995
 996         if (start_new) {
 997                 dep->resource_index = dwc3_gadget_ep_get_transfer_index(dwc,
 998                                 dep->number);
 999                 WARN_ON_ONCE(!dep->resource_index);
1000         }
1001
1002         return 0;
1003 }
1004
1005 static void __dwc3_gadget_start_isoc(struct dwc3 *dwc,
1006                 struct dwc3_ep *dep, u32 cur_uf)
1007 {
1008         u32 uf;
1009
1010         if (list_empty(&dep->request_list)) {
1011                 dev_vdbg(dwc->dev, "ISOC ep %s run out for requests.\n",
1012                         dep->name);
1013                 dep->flags |= DWC3_EP_PENDING_REQUEST;
1014                 return;
1015         }
1016
1017         /* 4 micro frames in the future */
1018         uf = cur_uf + dep->interval * 4;
1019
1020         __dwc3_gadget_kick_transfer(dep, uf, 1);
1021 }
1022
1023 static void dwc3_gadget_start_isoc(struct dwc3 *dwc,
1024                 struct dwc3_ep *dep, const struct dwc3_event_depevt *event)
1025 {
1026         u32 cur_uf, mask;
1027
1028         mask = ~(dep->interval - 1);
1029         cur_uf = event->parameters & mask;
1030
1031         __dwc3_gadget_start_isoc(dwc, dep, cur_uf);
1032 }
1033
1034 static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
1035 {
1036         struct dwc3             *dwc = dep->dwc;
1037         int                     ret;
1038
1039         req->request.actual     = 0;
1040         req->request.status     = -EINPROGRESS;
1041         req->direction          = dep->direction;
1042         req->epnum              = dep->number;
1043
1044         /*
1045          * We only add to our list of requests now and
1046          * start consuming the list once we get XferNotReady
1047          * IRQ.
1048          *
1049          * That way, we avoid doing anything that we don't need
1050          * to do now and defer it until the point we receive a
1051          * particular token from the Host side.
1052          *
1053          * This will also avoid Host cancelling URBs due to too
1054          * many NAKs.
1055          */
1056         ret = usb_gadget_map_request(&dwc->gadget, &req->request,
1057                         dep->direction);
1058         if (ret)
1059                 return ret;
1060
1061         list_add_tail(&req->list, &dep->request_list);
1062
1063         /*
1064          * There are a few special cases:
1065          *
1066          * 1. XferNotReady with empty list of requests. We need to kick the
1067          *    transfer here in that situation, otherwise we will be NAKing
1068          *    forever. If we get XferNotReady before gadget driver has a
1069          *    chance to queue a request, we will ACK the IRQ but won't be
1070          *    able to receive the data until the next request is queued.
1071          *    The following code is handling exactly that.
1072          *
1073          */
1074         if (dep->flags & DWC3_EP_PENDING_REQUEST) {
1075                 /*
1076                  * If xfernotready is already elapsed and it is a case
1077                  * of isoc transfer, then issue END TRANSFER, so that
1078                  * you can receive xfernotready again and can have
1079                  * notion of current microframe.
1080                  */
1081                 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1082                         if (list_empty(&dep->req_queued)) {
1083                                 dwc3_stop_active_transfer(dwc, dep->number);
1084                                 dep->flags = DWC3_EP_ENABLED;
1085                         }
1086                         return 0;
1087                 }
1088
1089                 ret = __dwc3_gadget_kick_transfer(dep, 0, true);
1090                 if (ret && ret != -EBUSY)
1091                         dev_dbg(dwc->dev, "%s: failed to kick transfers\n",
1092                                         dep->name);
1093                 return ret;
1094         }
1095
1096         /*
1097          * 2. XferInProgress on Isoc EP with an active transfer. We need to
1098          *    kick the transfer here after queuing a request, otherwise the
1099          *    core may not see the modified TRB(s).
1100          */
1101         if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
1102                         (dep->flags & DWC3_EP_BUSY) &&
1103                         !(dep->flags & DWC3_EP_MISSED_ISOC)) {
1104                 WARN_ON_ONCE(!dep->resource_index);
1105                 ret = __dwc3_gadget_kick_transfer(dep, dep->resource_index,
1106                                 false);
1107                 if (ret && ret != -EBUSY)
1108                         dev_dbg(dwc->dev, "%s: failed to kick transfers\n",
1109                                         dep->name);
1110                 return ret;
1111         }
1112
1113         return 0;
1114 }
1115
1116 static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
1117         gfp_t gfp_flags)
1118 {
1119         struct dwc3_request             *req = to_dwc3_request(request);
1120         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1121         struct dwc3                     *dwc = dep->dwc;
1122
1123         unsigned long                   flags;
1124
1125         int                             ret;
1126
1127         if (!dep->endpoint.desc) {
1128                 dev_dbg(dwc->dev, "trying to queue request %p to disabled %s\n",
1129                                 request, ep->name);
1130                 return -ESHUTDOWN;
1131         }
1132
1133         dev_vdbg(dwc->dev, "queing request %p to %s length %d\n",
1134                         request, ep->name, request->length);
1135
1136         spin_lock_irqsave(&dwc->lock, flags);
1137         ret = __dwc3_gadget_ep_queue(dep, req);
1138         spin_unlock_irqrestore(&dwc->lock, flags);
1139
1140         return ret;
1141 }
1142
1143 static int dwc3_gadget_ep_dequeue(struct usb_ep *ep,
1144                 struct usb_request *request)
1145 {
1146         struct dwc3_request             *req = to_dwc3_request(request);
1147         struct dwc3_request             *r = NULL;
1148
1149         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1150         struct dwc3                     *dwc = dep->dwc;
1151
1152         unsigned long                   flags;
1153         int                             ret = 0;
1154
1155         spin_lock_irqsave(&dwc->lock, flags);
1156
1157         list_for_each_entry(r, &dep->request_list, list) {
1158                 if (r == req)
1159                         break;
1160         }
1161
1162         if (r != req) {
1163                 list_for_each_entry(r, &dep->req_queued, list) {
1164                         if (r == req)
1165                                 break;
1166                 }
1167                 if (r == req) {
1168                         /* wait until it is processed */
1169                         dwc3_stop_active_transfer(dwc, dep->number);
1170                         goto out1;
1171                 }
1172                 dev_err(dwc->dev, "request %p was not queued to %s\n",
1173                                 request, ep->name);
1174                 ret = -EINVAL;
1175                 goto out0;
1176         }
1177
1178 out1:
1179         /* giveback the request */
1180         dwc3_gadget_giveback(dep, req, -ECONNRESET);
1181
1182 out0:
1183         spin_unlock_irqrestore(&dwc->lock, flags);
1184
1185         return ret;
1186 }
1187
1188 int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol)
1189 {
1190         struct dwc3_gadget_ep_cmd_params        params;
1191         struct dwc3                             *dwc = dep->dwc;
1192         int                                     ret;
1193
1194         memset(&params, 0x00, sizeof(params));
1195
1196         if (value) {
1197                 if (!protocol && ((dep->direction && dep->flags & DWC3_EP_BUSY) ||
1198                                 (!list_empty(&dep->req_queued) ||
1199                                  !list_empty(&dep->request_list)))) {
1200                         dev_dbg(dwc->dev, "%s: pending request, cannot halt\n",
1201                                         dep->name);
1202                         return -EAGAIN;
1203                 }
1204
1205                 ret = dwc3_send_gadget_ep_cmd(dwc, dep->number,
1206                         DWC3_DEPCMD_SETSTALL, &params);
1207                 if (ret)
1208                         dev_err(dwc->dev, "failed to %s STALL on %s\n",
1209                                         value ? "set" : "clear",
1210                                         dep->name);
1211                 else
1212                         dep->flags |= DWC3_EP_STALL;
1213         } else {
1214                 ret = dwc3_send_gadget_ep_cmd(dwc, dep->number,
1215                         DWC3_DEPCMD_CLEARSTALL, &params);
1216                 if (ret)
1217                         dev_err(dwc->dev, "failed to %s STALL on %s\n",
1218                                         value ? "set" : "clear",
1219                                         dep->name);
1220                 else
1221                         dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
1222         }
1223
1224         return ret;
1225 }
1226
1227 static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value)
1228 {
1229         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1230         struct dwc3                     *dwc = dep->dwc;
1231
1232         unsigned long                   flags;
1233
1234         int                             ret;
1235
1236         spin_lock_irqsave(&dwc->lock, flags);
1237
1238         if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1239                 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
1240                 ret = -EINVAL;
1241                 goto out;
1242         }
1243
1244         ret = __dwc3_gadget_ep_set_halt(dep, value, false);
1245 out:
1246         spin_unlock_irqrestore(&dwc->lock, flags);
1247
1248         return ret;
1249 }
1250
1251 static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep)
1252 {
1253         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1254         struct dwc3                     *dwc = dep->dwc;
1255         unsigned long                   flags;
1256
1257         spin_lock_irqsave(&dwc->lock, flags);
1258         dep->flags |= DWC3_EP_WEDGE;
1259         spin_unlock_irqrestore(&dwc->lock, flags);
1260
1261         if (dep->number == 0 || dep->number == 1)
1262                 return dwc3_gadget_ep0_set_halt(ep, 1);
1263         else
1264                 return __dwc3_gadget_ep_set_halt(dep, 1, false);
1265 }
1266
1267 /* -------------------------------------------------------------------------- */
1268
1269 static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = {
1270         .bLength        = USB_DT_ENDPOINT_SIZE,
1271         .bDescriptorType = USB_DT_ENDPOINT,
1272         .bmAttributes   = USB_ENDPOINT_XFER_CONTROL,
1273 };
1274
1275 static const struct usb_ep_ops dwc3_gadget_ep0_ops = {
1276         .enable         = dwc3_gadget_ep0_enable,
1277         .disable        = dwc3_gadget_ep0_disable,
1278         .alloc_request  = dwc3_gadget_ep_alloc_request,
1279         .free_request   = dwc3_gadget_ep_free_request,
1280         .queue          = dwc3_gadget_ep0_queue,
1281         .dequeue        = dwc3_gadget_ep_dequeue,
1282         .set_halt       = dwc3_gadget_ep0_set_halt,
1283         .set_wedge      = dwc3_gadget_ep_set_wedge,
1284 };
1285
1286 static const struct usb_ep_ops dwc3_gadget_ep_ops = {
1287         .enable         = dwc3_gadget_ep_enable,
1288         .disable        = dwc3_gadget_ep_disable,
1289         .alloc_request  = dwc3_gadget_ep_alloc_request,
1290         .free_request   = dwc3_gadget_ep_free_request,
1291         .queue          = dwc3_gadget_ep_queue,
1292         .dequeue        = dwc3_gadget_ep_dequeue,
1293         .set_halt       = dwc3_gadget_ep_set_halt,
1294         .set_wedge      = dwc3_gadget_ep_set_wedge,
1295 };
1296
1297 /* -------------------------------------------------------------------------- */
1298
1299 static int dwc3_gadget_get_frame(struct usb_gadget *g)
1300 {
1301         struct dwc3             *dwc = gadget_to_dwc(g);
1302         u32                     reg;
1303
1304         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1305         return DWC3_DSTS_SOFFN(reg);
1306 }
1307
1308 static int dwc3_gadget_wakeup(struct usb_gadget *g)
1309 {
1310         struct dwc3             *dwc = gadget_to_dwc(g);
1311
1312         unsigned long           timeout;
1313         unsigned long           flags;
1314
1315         u32                     reg;
1316
1317         int                     ret = 0;
1318
1319         u8                      link_state;
1320         u8                      speed;
1321
1322         spin_lock_irqsave(&dwc->lock, flags);
1323
1324         /*
1325          * According to the Databook Remote wakeup request should
1326          * be issued only when the device is in early suspend state.
1327          *
1328          * We can check that via USB Link State bits in DSTS register.
1329          */
1330         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1331
1332         speed = reg & DWC3_DSTS_CONNECTSPD;
1333         if (speed == DWC3_DSTS_SUPERSPEED) {
1334                 dev_dbg(dwc->dev, "no wakeup on SuperSpeed\n");
1335                 ret = -EINVAL;
1336                 goto out;
1337         }
1338
1339         link_state = DWC3_DSTS_USBLNKST(reg);
1340
1341         switch (link_state) {
1342         case DWC3_LINK_STATE_RX_DET:    /* in HS, means Early Suspend */
1343         case DWC3_LINK_STATE_U3:        /* in HS, means SUSPEND */
1344                 break;
1345         default:
1346                 dev_dbg(dwc->dev, "can't wakeup from link state %d\n",
1347                                 link_state);
1348                 ret = -EINVAL;
1349                 goto out;
1350         }
1351
1352         ret = dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RECOV);
1353         if (ret < 0) {
1354                 dev_err(dwc->dev, "failed to put link in Recovery\n");
1355                 goto out;
1356         }
1357
1358         /* Recent versions do this automatically */
1359         if (dwc->revision < DWC3_REVISION_194A) {
1360                 /* write zeroes to Link Change Request */
1361                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1362                 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
1363                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1364         }
1365
1366         /* poll until Link State changes to ON */
1367         timeout = jiffies + msecs_to_jiffies(100);
1368
1369         while (!time_after(jiffies, timeout)) {
1370                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1371
1372                 /* in HS, means ON */
1373                 if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0)
1374                         break;
1375         }
1376
1377         if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) {
1378                 dev_err(dwc->dev, "failed to send remote wakeup\n");
1379                 ret = -EINVAL;
1380         }
1381
1382 out:
1383         spin_unlock_irqrestore(&dwc->lock, flags);
1384
1385         return ret;
1386 }
1387
1388 static int dwc3_gadget_set_selfpowered(struct usb_gadget *g,
1389                 int is_selfpowered)
1390 {
1391         struct dwc3             *dwc = gadget_to_dwc(g);
1392         unsigned long           flags;
1393
1394         spin_lock_irqsave(&dwc->lock, flags);
1395         dwc->is_selfpowered = !!is_selfpowered;
1396         spin_unlock_irqrestore(&dwc->lock, flags);
1397
1398         return 0;
1399 }
1400
1401 static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on)
1402 {
1403         u32                     reg;
1404         u32                     timeout = 500;
1405
1406         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1407         if (is_on) {
1408                 if (dwc->revision <= DWC3_REVISION_187A) {
1409                         reg &= ~DWC3_DCTL_TRGTULST_MASK;
1410                         reg |= DWC3_DCTL_TRGTULST_RX_DET;
1411                 }
1412
1413                 if (dwc->revision >= DWC3_REVISION_194A)
1414                         reg &= ~DWC3_DCTL_KEEP_CONNECT;
1415                 reg |= DWC3_DCTL_RUN_STOP;
1416                 dwc->pullups_connected = true;
1417         } else {
1418                 reg &= ~DWC3_DCTL_RUN_STOP;
1419                 dwc->pullups_connected = false;
1420         }
1421
1422         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1423
1424         do {
1425                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1426                 if (is_on) {
1427                         if (!(reg & DWC3_DSTS_DEVCTRLHLT))
1428                                 break;
1429                 } else {
1430                         if (reg & DWC3_DSTS_DEVCTRLHLT)
1431                                 break;
1432                 }
1433                 timeout--;
1434                 if (!timeout)
1435                         return -ETIMEDOUT;
1436                 udelay(1);
1437         } while (1);
1438
1439         dev_vdbg(dwc->dev, "gadget %s data soft-%s\n",
1440                         dwc->gadget_driver
1441                         ? dwc->gadget_driver->function : "no-function",
1442                         is_on ? "connect" : "disconnect");
1443
1444         return 0;
1445 }
1446
1447 static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
1448 {
1449         struct dwc3             *dwc = gadget_to_dwc(g);
1450         unsigned long           flags;
1451         int                     ret;
1452
1453         is_on = !!is_on;
1454
1455         spin_lock_irqsave(&dwc->lock, flags);
1456         ret = dwc3_gadget_run_stop(dwc, is_on);
1457         spin_unlock_irqrestore(&dwc->lock, flags);
1458
1459         return ret;
1460 }
1461
1462 static void dwc3_gadget_enable_irq(struct dwc3 *dwc)
1463 {
1464         u32                     reg;
1465
1466         /* Enable all but Start and End of Frame IRQs */
1467         reg = (DWC3_DEVTEN_VNDRDEVTSTRCVEDEN |
1468                         DWC3_DEVTEN_EVNTOVERFLOWEN |
1469                         DWC3_DEVTEN_CMDCMPLTEN |
1470                         DWC3_DEVTEN_ERRTICERREN |
1471                         DWC3_DEVTEN_WKUPEVTEN |
1472                         DWC3_DEVTEN_ULSTCNGEN |
1473                         DWC3_DEVTEN_CONNECTDONEEN |
1474                         DWC3_DEVTEN_USBRSTEN |
1475                         DWC3_DEVTEN_DISCONNEVTEN);
1476
1477         dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
1478 }
1479
1480 static void dwc3_gadget_disable_irq(struct dwc3 *dwc)
1481 {
1482         /* mask all interrupts */
1483         dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);
1484 }
1485
1486 static irqreturn_t dwc3_interrupt(int irq, void *_dwc);
1487 static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc);
1488
1489 static int dwc3_gadget_start(struct usb_gadget *g,
1490                 struct usb_gadget_driver *driver)
1491 {
1492         struct dwc3             *dwc = gadget_to_dwc(g);
1493         struct dwc3_ep          *dep;
1494         unsigned long           flags;
1495         int                     ret = 0;
1496         int                     irq;
1497         u32                     reg;
1498
1499         irq = platform_get_irq(to_platform_device(dwc->dev), 0);
1500         ret = request_threaded_irq(irq, dwc3_interrupt, dwc3_thread_interrupt,
1501                         IRQF_SHARED, "dwc3", dwc);
1502         if (ret) {
1503                 dev_err(dwc->dev, "failed to request irq #%d --> %d\n",
1504                                 irq, ret);
1505                 goto err0;
1506         }
1507
1508         spin_lock_irqsave(&dwc->lock, flags);
1509
1510         if (dwc->gadget_driver) {
1511                 dev_err(dwc->dev, "%s is already bound to %s\n",
1512                                 dwc->gadget.name,
1513                                 dwc->gadget_driver->driver.name);
1514                 ret = -EBUSY;
1515                 goto err1;
1516         }
1517
1518         dwc->gadget_driver      = driver;
1519
1520         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
1521         reg &= ~(DWC3_DCFG_SPEED_MASK);
1522
1523         /**
1524          * WORKAROUND: DWC3 revision < 2.20a have an issue
1525          * which would cause metastability state on Run/Stop
1526          * bit if we try to force the IP to USB2-only mode.
1527          *
1528          * Because of that, we cannot configure the IP to any
1529          * speed other than the SuperSpeed
1530          *
1531          * Refers to:
1532          *
1533          * STAR#9000525659: Clock Domain Crossing on DCTL in
1534          * USB 2.0 Mode
1535          */
1536         if (dwc->revision < DWC3_REVISION_220A) {
1537                 reg |= DWC3_DCFG_SUPERSPEED;
1538         } else {
1539                 switch (dwc->maximum_speed) {
1540                 case USB_SPEED_LOW:
1541                         reg |= DWC3_DSTS_LOWSPEED;
1542                         break;
1543                 case USB_SPEED_FULL:
1544                         reg |= DWC3_DSTS_FULLSPEED1;
1545                         break;
1546                 case USB_SPEED_HIGH:
1547                         reg |= DWC3_DSTS_HIGHSPEED;
1548                         break;
1549                 case USB_SPEED_SUPER:   /* FALLTHROUGH */
1550                 case USB_SPEED_UNKNOWN: /* FALTHROUGH */
1551                 default:
1552                         reg |= DWC3_DSTS_SUPERSPEED;
1553                 }
1554         }
1555         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
1556
1557         dwc->start_config_issued = false;
1558
1559         /* Start with SuperSpeed Default */
1560         dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1561
1562         dep = dwc->eps[0];
1563         ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false);
1564         if (ret) {
1565                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1566                 goto err2;
1567         }
1568
1569         dep = dwc->eps[1];
1570         ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false);
1571         if (ret) {
1572                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1573                 goto err3;
1574         }
1575
1576         /* begin to receive SETUP packets */
1577         dwc->ep0state = EP0_SETUP_PHASE;
1578         dwc3_ep0_out_start(dwc);
1579
1580         dwc3_gadget_enable_irq(dwc);
1581
1582         spin_unlock_irqrestore(&dwc->lock, flags);
1583
1584         return 0;
1585
1586 err3:
1587         __dwc3_gadget_ep_disable(dwc->eps[0]);
1588
1589 err2:
1590         dwc->gadget_driver = NULL;
1591
1592 err1:
1593         spin_unlock_irqrestore(&dwc->lock, flags);
1594
1595         free_irq(irq, dwc);
1596
1597 err0:
1598         return ret;
1599 }
1600
1601 static int dwc3_gadget_stop(struct usb_gadget *g,
1602                 struct usb_gadget_driver *driver)
1603 {
1604         struct dwc3             *dwc = gadget_to_dwc(g);
1605         unsigned long           flags;
1606         int                     irq;
1607
1608         spin_lock_irqsave(&dwc->lock, flags);
1609
1610         dwc3_gadget_disable_irq(dwc);
1611         __dwc3_gadget_ep_disable(dwc->eps[0]);
1612         __dwc3_gadget_ep_disable(dwc->eps[1]);
1613
1614         dwc->gadget_driver      = NULL;
1615
1616         spin_unlock_irqrestore(&dwc->lock, flags);
1617
1618         irq = platform_get_irq(to_platform_device(dwc->dev), 0);
1619         free_irq(irq, dwc);
1620
1621         return 0;
1622 }
1623
1624 static const struct usb_gadget_ops dwc3_gadget_ops = {
1625         .get_frame              = dwc3_gadget_get_frame,
1626         .wakeup                 = dwc3_gadget_wakeup,
1627         .set_selfpowered        = dwc3_gadget_set_selfpowered,
1628         .pullup                 = dwc3_gadget_pullup,
1629         .udc_start              = dwc3_gadget_start,
1630         .udc_stop               = dwc3_gadget_stop,
1631 };
1632
1633 /* -------------------------------------------------------------------------- */
1634
1635 static int dwc3_gadget_init_hw_endpoints(struct dwc3 *dwc,
1636                 u8 num, u32 direction)
1637 {
1638         struct dwc3_ep                  *dep;
1639         u8                              i;
1640
1641         for (i = 0; i < num; i++) {
1642                 u8 epnum = (i << 1) | (!!direction);
1643
1644                 dep = kzalloc(sizeof(*dep), GFP_KERNEL);
1645                 if (!dep) {
1646                         dev_err(dwc->dev, "can't allocate endpoint %d\n",
1647                                         epnum);
1648                         return -ENOMEM;
1649                 }
1650
1651                 dep->dwc = dwc;
1652                 dep->number = epnum;
1653                 dep->direction = !!direction;
1654                 dwc->eps[epnum] = dep;
1655
1656                 snprintf(dep->name, sizeof(dep->name), "ep%d%s", epnum >> 1,
1657                                 (epnum & 1) ? "in" : "out");
1658
1659                 dep->endpoint.name = dep->name;
1660
1661                 dev_vdbg(dwc->dev, "initializing %s\n", dep->name);
1662
1663                 if (epnum == 0 || epnum == 1) {
1664                         usb_ep_set_maxpacket_limit(&dep->endpoint, 512);
1665                         dep->endpoint.maxburst = 1;
1666                         dep->endpoint.ops = &dwc3_gadget_ep0_ops;
1667                         if (!epnum)
1668                                 dwc->gadget.ep0 = &dep->endpoint;
1669                 } else {
1670                         int             ret;
1671
1672                         usb_ep_set_maxpacket_limit(&dep->endpoint, 1024);
1673                         dep->endpoint.max_streams = 15;
1674                         dep->endpoint.ops = &dwc3_gadget_ep_ops;
1675                         list_add_tail(&dep->endpoint.ep_list,
1676                                         &dwc->gadget.ep_list);
1677
1678                         ret = dwc3_alloc_trb_pool(dep);
1679                         if (ret)
1680                                 return ret;
1681                 }
1682
1683                 INIT_LIST_HEAD(&dep->request_list);
1684                 INIT_LIST_HEAD(&dep->req_queued);
1685         }
1686
1687         return 0;
1688 }
1689
1690 static int dwc3_gadget_init_endpoints(struct dwc3 *dwc)
1691 {
1692         int                             ret;
1693
1694         INIT_LIST_HEAD(&dwc->gadget.ep_list);
1695
1696         ret = dwc3_gadget_init_hw_endpoints(dwc, dwc->num_out_eps, 0);
1697         if (ret < 0) {
1698                 dev_vdbg(dwc->dev, "failed to allocate OUT endpoints\n");
1699                 return ret;
1700         }
1701
1702         ret = dwc3_gadget_init_hw_endpoints(dwc, dwc->num_in_eps, 1);
1703         if (ret < 0) {
1704                 dev_vdbg(dwc->dev, "failed to allocate IN endpoints\n");
1705                 return ret;
1706         }
1707
1708         return 0;
1709 }
1710
1711 static void dwc3_gadget_free_endpoints(struct dwc3 *dwc)
1712 {
1713         struct dwc3_ep                  *dep;
1714         u8                              epnum;
1715
1716         for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
1717                 dep = dwc->eps[epnum];
1718                 if (!dep)
1719                         continue;
1720                 /*
1721                  * Physical endpoints 0 and 1 are special; they form the
1722                  * bi-directional USB endpoint 0.
1723                  *
1724                  * For those two physical endpoints, we don't allocate a TRB
1725                  * pool nor do we add them the endpoints list. Due to that, we
1726                  * shouldn't do these two operations otherwise we would end up
1727                  * with all sorts of bugs when removing dwc3.ko.
1728                  */
1729                 if (epnum != 0 && epnum != 1) {
1730                         dwc3_free_trb_pool(dep);
1731                         list_del(&dep->endpoint.ep_list);
1732                 }
1733
1734                 kfree(dep);
1735         }
1736 }
1737
1738 /* -------------------------------------------------------------------------- */
1739
1740 static int __dwc3_cleanup_done_trbs(struct dwc3 *dwc, struct dwc3_ep *dep,
1741                 struct dwc3_request *req, struct dwc3_trb *trb,
1742                 const struct dwc3_event_depevt *event, int status)
1743 {
1744         unsigned int            count;
1745         unsigned int            s_pkt = 0;
1746         unsigned int            trb_status;
1747
1748         if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN)
1749                 /*
1750                  * We continue despite the error. There is not much we
1751                  * can do. If we don't clean it up we loop forever. If
1752                  * we skip the TRB then it gets overwritten after a
1753                  * while since we use them in a ring buffer. A BUG()
1754                  * would help. Lets hope that if this occurs, someone
1755                  * fixes the root cause instead of looking away :)
1756                  */
1757                 dev_err(dwc->dev, "%s's TRB (%p) still owned by HW\n",
1758                                 dep->name, trb);
1759         count = trb->size & DWC3_TRB_SIZE_MASK;
1760
1761         if (dep->direction) {
1762                 if (count) {
1763                         trb_status = DWC3_TRB_SIZE_TRBSTS(trb->size);
1764                         if (trb_status == DWC3_TRBSTS_MISSED_ISOC) {
1765                                 dev_dbg(dwc->dev, "incomplete IN transfer %s\n",
1766                                                 dep->name);
1767                                 /*
1768                                  * If missed isoc occurred and there is
1769                                  * no request queued then issue END
1770                                  * TRANSFER, so that core generates
1771                                  * next xfernotready and we will issue
1772                                  * a fresh START TRANSFER.
1773                                  * If there are still queued request
1774                                  * then wait, do not issue either END
1775                                  * or UPDATE TRANSFER, just attach next
1776                                  * request in request_list during
1777                                  * giveback.If any future queued request
1778                                  * is successfully transferred then we
1779                                  * will issue UPDATE TRANSFER for all
1780                                  * request in the request_list.
1781                                  */
1782                                 dep->flags |= DWC3_EP_MISSED_ISOC;
1783                         } else {
1784                                 dev_err(dwc->dev, "incomplete IN transfer %s\n",
1785                                                 dep->name);
1786                                 status = -ECONNRESET;
1787                         }
1788                 } else {
1789                         dep->flags &= ~DWC3_EP_MISSED_ISOC;
1790                 }
1791         } else {
1792                 if (count && (event->status & DEPEVT_STATUS_SHORT))
1793                         s_pkt = 1;
1794         }
1795
1796         /*
1797          * We assume here we will always receive the entire data block
1798          * which we should receive. Meaning, if we program RX to
1799          * receive 4K but we receive only 2K, we assume that's all we
1800          * should receive and we simply bounce the request back to the
1801          * gadget driver for further processing.
1802          */
1803         req->request.actual += req->request.length - count;
1804         if (s_pkt)
1805                 return 1;
1806         if ((event->status & DEPEVT_STATUS_LST) &&
1807                         (trb->ctrl & (DWC3_TRB_CTRL_LST |
1808                                 DWC3_TRB_CTRL_HWO)))
1809                 return 1;
1810         if ((event->status & DEPEVT_STATUS_IOC) &&
1811                         (trb->ctrl & DWC3_TRB_CTRL_IOC))
1812                 return 1;
1813         return 0;
1814 }
1815
1816 static int dwc3_cleanup_done_reqs(struct dwc3 *dwc, struct dwc3_ep *dep,
1817                 const struct dwc3_event_depevt *event, int status)
1818 {
1819         struct dwc3_request     *req;
1820         struct dwc3_trb         *trb;
1821         unsigned int            slot;
1822         unsigned int            i;
1823         int                     ret;
1824
1825         do {
1826                 req = next_request(&dep->req_queued);
1827                 if (!req) {
1828                         WARN_ON_ONCE(1);
1829                         return 1;
1830                 }
1831                 i = 0;
1832                 do {
1833                         slot = req->start_slot + i;
1834                         if ((slot == DWC3_TRB_NUM - 1) &&
1835                                 usb_endpoint_xfer_isoc(dep->endpoint.desc))
1836                                 slot++;
1837                         slot %= DWC3_TRB_NUM;
1838                         trb = &dep->trb_pool[slot];
1839
1840                         ret = __dwc3_cleanup_done_trbs(dwc, dep, req, trb,
1841                                         event, status);
1842                         if (ret)
1843                                 break;
1844                 }while (++i < req->request.num_mapped_sgs);
1845
1846                 dwc3_gadget_giveback(dep, req, status);
1847
1848                 if (ret)
1849                         break;
1850         } while (1);
1851
1852         if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
1853                         list_empty(&dep->req_queued)) {
1854                 if (list_empty(&dep->request_list)) {
1855                         /*
1856                          * If there is no entry in request list then do
1857                          * not issue END TRANSFER now. Just set PENDING
1858                          * flag, so that END TRANSFER is issued when an
1859                          * entry is added into request list.
1860                          */
1861                         dep->flags = DWC3_EP_PENDING_REQUEST;
1862                 } else {
1863                         dwc3_stop_active_transfer(dwc, dep->number);
1864                         dep->flags = DWC3_EP_ENABLED;
1865                 }
1866                 return 1;
1867         }
1868
1869         if ((event->status & DEPEVT_STATUS_IOC) &&
1870                         (trb->ctrl & DWC3_TRB_CTRL_IOC))
1871                 return 0;
1872         return 1;
1873 }
1874
1875 static void dwc3_endpoint_transfer_complete(struct dwc3 *dwc,
1876                 struct dwc3_ep *dep, const struct dwc3_event_depevt *event,
1877                 int start_new)
1878 {
1879         unsigned                status = 0;
1880         int                     clean_busy;
1881
1882         if (event->status & DEPEVT_STATUS_BUSERR)
1883                 status = -ECONNRESET;
1884
1885         clean_busy = dwc3_cleanup_done_reqs(dwc, dep, event, status);
1886         if (clean_busy)
1887                 dep->flags &= ~DWC3_EP_BUSY;
1888
1889         /*
1890          * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround.
1891          * See dwc3_gadget_linksts_change_interrupt() for 1st half.
1892          */
1893         if (dwc->revision < DWC3_REVISION_183A) {
1894                 u32             reg;
1895                 int             i;
1896
1897                 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
1898                         dep = dwc->eps[i];
1899
1900                         if (!(dep->flags & DWC3_EP_ENABLED))
1901                                 continue;
1902
1903                         if (!list_empty(&dep->req_queued))
1904                                 return;
1905                 }
1906
1907                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1908                 reg |= dwc->u1u2;
1909                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1910
1911                 dwc->u1u2 = 0;
1912         }
1913 }
1914
1915 static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
1916                 const struct dwc3_event_depevt *event)
1917 {
1918         struct dwc3_ep          *dep;
1919         u8                      epnum = event->endpoint_number;
1920
1921         dep = dwc->eps[epnum];
1922
1923         if (!(dep->flags & DWC3_EP_ENABLED))
1924                 return;
1925
1926         dev_vdbg(dwc->dev, "%s: %s\n", dep->name,
1927                         dwc3_ep_event_string(event->endpoint_event));
1928
1929         if (epnum == 0 || epnum == 1) {
1930                 dwc3_ep0_interrupt(dwc, event);
1931                 return;
1932         }
1933
1934         switch (event->endpoint_event) {
1935         case DWC3_DEPEVT_XFERCOMPLETE:
1936                 dep->resource_index = 0;
1937
1938                 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1939                         dev_dbg(dwc->dev, "%s is an Isochronous endpoint\n",
1940                                         dep->name);
1941                         return;
1942                 }
1943
1944                 dwc3_endpoint_transfer_complete(dwc, dep, event, 1);
1945                 break;
1946         case DWC3_DEPEVT_XFERINPROGRESS:
1947                 if (!usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1948                         dev_dbg(dwc->dev, "%s is not an Isochronous endpoint\n",
1949                                         dep->name);
1950                         return;
1951                 }
1952
1953                 dwc3_endpoint_transfer_complete(dwc, dep, event, 0);
1954                 break;
1955         case DWC3_DEPEVT_XFERNOTREADY:
1956                 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1957                         dwc3_gadget_start_isoc(dwc, dep, event);
1958                 } else {
1959                         int ret;
1960
1961                         dev_vdbg(dwc->dev, "%s: reason %s\n",
1962                                         dep->name, event->status &
1963                                         DEPEVT_STATUS_TRANSFER_ACTIVE
1964                                         ? "Transfer Active"
1965                                         : "Transfer Not Active");
1966
1967                         ret = __dwc3_gadget_kick_transfer(dep, 0, 1);
1968                         if (!ret || ret == -EBUSY)
1969                                 return;
1970
1971                         dev_dbg(dwc->dev, "%s: failed to kick transfers\n",
1972                                         dep->name);
1973                 }
1974
1975                 break;
1976         case DWC3_DEPEVT_STREAMEVT:
1977                 if (!usb_endpoint_xfer_bulk(dep->endpoint.desc)) {
1978                         dev_err(dwc->dev, "Stream event for non-Bulk %s\n",
1979                                         dep->name);
1980                         return;
1981                 }
1982
1983                 switch (event->status) {
1984                 case DEPEVT_STREAMEVT_FOUND:
1985                         dev_vdbg(dwc->dev, "Stream %d found and started\n",
1986                                         event->parameters);
1987
1988                         break;
1989                 case DEPEVT_STREAMEVT_NOTFOUND:
1990                         /* FALLTHROUGH */
1991                 default:
1992                         dev_dbg(dwc->dev, "Couldn't find suitable stream\n");
1993                 }
1994                 break;
1995         case DWC3_DEPEVT_RXTXFIFOEVT:
1996                 dev_dbg(dwc->dev, "%s FIFO Overrun\n", dep->name);
1997                 break;
1998         case DWC3_DEPEVT_EPCMDCMPLT:
1999                 dev_vdbg(dwc->dev, "Endpoint Command Complete\n");
2000                 break;
2001         }
2002 }
2003
2004 static void dwc3_disconnect_gadget(struct dwc3 *dwc)
2005 {
2006         if (dwc->gadget_driver && dwc->gadget_driver->disconnect) {
2007                 spin_unlock(&dwc->lock);
2008                 dwc->gadget_driver->disconnect(&dwc->gadget);
2009                 spin_lock(&dwc->lock);
2010         }
2011 }
2012
2013 static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum)
2014 {
2015         struct dwc3_ep *dep;
2016         struct dwc3_gadget_ep_cmd_params params;
2017         u32 cmd;
2018         int ret;
2019
2020         dep = dwc->eps[epnum];
2021
2022         if (!dep->resource_index)
2023                 return;
2024
2025         /*
2026          * NOTICE: We are violating what the Databook says about the
2027          * EndTransfer command. Ideally we would _always_ wait for the
2028          * EndTransfer Command Completion IRQ, but that's causing too
2029          * much trouble synchronizing between us and gadget driver.
2030          *
2031          * We have discussed this with the IP Provider and it was
2032          * suggested to giveback all requests here, but give HW some
2033          * extra time to synchronize with the interconnect. We're using
2034          * an arbitraty 100us delay for that.
2035          *
2036          * Note also that a similar handling was tested by Synopsys
2037          * (thanks a lot Paul) and nothing bad has come out of it.
2038          * In short, what we're doing is:
2039          *
2040          * - Issue EndTransfer WITH CMDIOC bit set
2041          * - Wait 100us
2042          */
2043
2044         cmd = DWC3_DEPCMD_ENDTRANSFER;
2045         cmd |= DWC3_DEPCMD_HIPRI_FORCERM | DWC3_DEPCMD_CMDIOC;
2046         cmd |= DWC3_DEPCMD_PARAM(dep->resource_index);
2047         memset(&params, 0, sizeof(params));
2048         ret = dwc3_send_gadget_ep_cmd(dwc, dep->number, cmd, &params);
2049         WARN_ON_ONCE(ret);
2050         dep->resource_index = 0;
2051         dep->flags &= ~DWC3_EP_BUSY;
2052         udelay(100);
2053 }
2054
2055 static void dwc3_stop_active_transfers(struct dwc3 *dwc)
2056 {
2057         u32 epnum;
2058
2059         for (epnum = 2; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2060                 struct dwc3_ep *dep;
2061
2062                 dep = dwc->eps[epnum];
2063                 if (!dep)
2064                         continue;
2065
2066                 if (!(dep->flags & DWC3_EP_ENABLED))
2067                         continue;
2068
2069                 dwc3_remove_requests(dwc, dep);
2070         }
2071 }
2072
2073 static void dwc3_clear_stall_all_ep(struct dwc3 *dwc)
2074 {
2075         u32 epnum;
2076
2077         for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2078                 struct dwc3_ep *dep;
2079                 struct dwc3_gadget_ep_cmd_params params;
2080                 int ret;
2081
2082                 dep = dwc->eps[epnum];
2083                 if (!dep)
2084                         continue;
2085
2086                 if (!(dep->flags & DWC3_EP_STALL))
2087                         continue;
2088
2089                 dep->flags &= ~DWC3_EP_STALL;
2090
2091                 memset(&params, 0, sizeof(params));
2092                 ret = dwc3_send_gadget_ep_cmd(dwc, dep->number,
2093                                 DWC3_DEPCMD_CLEARSTALL, &params);
2094                 WARN_ON_ONCE(ret);
2095         }
2096 }
2097
2098 static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc)
2099 {
2100         int                     reg;
2101
2102         dev_vdbg(dwc->dev, "%s\n", __func__);
2103
2104         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2105         reg &= ~DWC3_DCTL_INITU1ENA;
2106         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2107
2108         reg &= ~DWC3_DCTL_INITU2ENA;
2109         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2110
2111         dwc3_disconnect_gadget(dwc);
2112         dwc->start_config_issued = false;
2113
2114         dwc->gadget.speed = USB_SPEED_UNKNOWN;
2115         dwc->setup_packet_pending = false;
2116 }
2117
2118 static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
2119 {
2120         u32                     reg;
2121
2122         dev_vdbg(dwc->dev, "%s\n", __func__);
2123
2124         /*
2125          * WORKAROUND: DWC3 revisions <1.88a have an issue which
2126          * would cause a missing Disconnect Event if there's a
2127          * pending Setup Packet in the FIFO.
2128          *
2129          * There's no suggested workaround on the official Bug
2130          * report, which states that "unless the driver/application
2131          * is doing any special handling of a disconnect event,
2132          * there is no functional issue".
2133          *
2134          * Unfortunately, it turns out that we _do_ some special
2135          * handling of a disconnect event, namely complete all
2136          * pending transfers, notify gadget driver of the
2137          * disconnection, and so on.
2138          *
2139          * Our suggested workaround is to follow the Disconnect
2140          * Event steps here, instead, based on a setup_packet_pending
2141          * flag. Such flag gets set whenever we have a XferNotReady
2142          * event on EP0 and gets cleared on XferComplete for the
2143          * same endpoint.
2144          *
2145          * Refers to:
2146          *
2147          * STAR#9000466709: RTL: Device : Disconnect event not
2148          * generated if setup packet pending in FIFO
2149          */
2150         if (dwc->revision < DWC3_REVISION_188A) {
2151                 if (dwc->setup_packet_pending)
2152                         dwc3_gadget_disconnect_interrupt(dwc);
2153         }
2154
2155         /* after reset -> Default State */
2156         usb_gadget_set_state(&dwc->gadget, USB_STATE_DEFAULT);
2157
2158         if (dwc->gadget.speed != USB_SPEED_UNKNOWN)
2159                 dwc3_disconnect_gadget(dwc);
2160
2161         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2162         reg &= ~DWC3_DCTL_TSTCTRL_MASK;
2163         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2164         dwc->test_mode = false;
2165
2166         dwc3_stop_active_transfers(dwc);
2167         dwc3_clear_stall_all_ep(dwc);
2168         dwc->start_config_issued = false;
2169
2170         /* Reset device address to zero */
2171         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2172         reg &= ~(DWC3_DCFG_DEVADDR_MASK);
2173         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2174 }
2175
2176 static void dwc3_update_ram_clk_sel(struct dwc3 *dwc, u32 speed)
2177 {
2178         u32 reg;
2179         u32 usb30_clock = DWC3_GCTL_CLK_BUS;
2180
2181         /*
2182          * We change the clock only at SS but I dunno why I would want to do
2183          * this. Maybe it becomes part of the power saving plan.
2184          */
2185
2186         if (speed != DWC3_DSTS_SUPERSPEED)
2187                 return;
2188
2189         /*
2190          * RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed
2191          * each time on Connect Done.
2192          */
2193         if (!usb30_clock)
2194                 return;
2195
2196         reg = dwc3_readl(dwc->regs, DWC3_GCTL);
2197         reg |= DWC3_GCTL_RAMCLKSEL(usb30_clock);
2198         dwc3_writel(dwc->regs, DWC3_GCTL, reg);
2199 }
2200
2201 static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
2202 {
2203         struct dwc3_ep          *dep;
2204         int                     ret;
2205         u32                     reg;
2206         u8                      speed;
2207
2208         dev_vdbg(dwc->dev, "%s\n", __func__);
2209
2210         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2211         speed = reg & DWC3_DSTS_CONNECTSPD;
2212         dwc->speed = speed;
2213
2214         dwc3_update_ram_clk_sel(dwc, speed);
2215
2216         switch (speed) {
2217         case DWC3_DCFG_SUPERSPEED:
2218                 /*
2219                  * WORKAROUND: DWC3 revisions <1.90a have an issue which
2220                  * would cause a missing USB3 Reset event.
2221                  *
2222                  * In such situations, we should force a USB3 Reset
2223                  * event by calling our dwc3_gadget_reset_interrupt()
2224                  * routine.
2225                  *
2226                  * Refers to:
2227                  *
2228                  * STAR#9000483510: RTL: SS : USB3 reset event may
2229                  * not be generated always when the link enters poll
2230                  */
2231                 if (dwc->revision < DWC3_REVISION_190A)
2232                         dwc3_gadget_reset_interrupt(dwc);
2233
2234                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2235                 dwc->gadget.ep0->maxpacket = 512;
2236                 dwc->gadget.speed = USB_SPEED_SUPER;
2237                 break;
2238         case DWC3_DCFG_HIGHSPEED:
2239                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
2240                 dwc->gadget.ep0->maxpacket = 64;
2241                 dwc->gadget.speed = USB_SPEED_HIGH;
2242                 break;
2243         case DWC3_DCFG_FULLSPEED2:
2244         case DWC3_DCFG_FULLSPEED1:
2245                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
2246                 dwc->gadget.ep0->maxpacket = 64;
2247                 dwc->gadget.speed = USB_SPEED_FULL;
2248                 break;
2249         case DWC3_DCFG_LOWSPEED:
2250                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(8);
2251                 dwc->gadget.ep0->maxpacket = 8;
2252                 dwc->gadget.speed = USB_SPEED_LOW;
2253                 break;
2254         }
2255
2256         /* Enable USB2 LPM Capability */
2257
2258         if ((dwc->revision > DWC3_REVISION_194A)
2259                         && (speed != DWC3_DCFG_SUPERSPEED)) {
2260                 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2261                 reg |= DWC3_DCFG_LPM_CAP;
2262                 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2263
2264                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2265                 reg &= ~(DWC3_DCTL_HIRD_THRES_MASK | DWC3_DCTL_L1_HIBER_EN);
2266
2267                 /*
2268                  * TODO: This should be configurable. For now using
2269                  * maximum allowed HIRD threshold value of 0b1100
2270                  */
2271                 reg |= DWC3_DCTL_HIRD_THRES(12);
2272
2273                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2274         }
2275
2276         dep = dwc->eps[0];
2277         ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, true);
2278         if (ret) {
2279                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2280                 return;
2281         }
2282
2283         dep = dwc->eps[1];
2284         ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, true);
2285         if (ret) {
2286                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2287                 return;
2288         }
2289
2290         /*
2291          * Configure PHY via GUSB3PIPECTLn if required.
2292          *
2293          * Update GTXFIFOSIZn
2294          *
2295          * In both cases reset values should be sufficient.
2296          */
2297 }
2298
2299 static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc)
2300 {
2301         dev_vdbg(dwc->dev, "%s\n", __func__);
2302
2303         /*
2304          * TODO take core out of low power mode when that's
2305          * implemented.
2306          */
2307
2308         dwc->gadget_driver->resume(&dwc->gadget);
2309 }
2310
2311 static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
2312                 unsigned int evtinfo)
2313 {
2314         enum dwc3_link_state    next = evtinfo & DWC3_LINK_STATE_MASK;
2315         unsigned int            pwropt;
2316
2317         /*
2318          * WORKAROUND: DWC3 < 2.50a have an issue when configured without
2319          * Hibernation mode enabled which would show up when device detects
2320          * host-initiated U3 exit.
2321          *
2322          * In that case, device will generate a Link State Change Interrupt
2323          * from U3 to RESUME which is only necessary if Hibernation is
2324          * configured in.
2325          *
2326          * There are no functional changes due to such spurious event and we
2327          * just need to ignore it.
2328          *
2329          * Refers to:
2330          *
2331          * STAR#9000570034 RTL: SS Resume event generated in non-Hibernation
2332          * operational mode
2333          */
2334         pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1);
2335         if ((dwc->revision < DWC3_REVISION_250A) &&
2336                         (pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) {
2337                 if ((dwc->link_state == DWC3_LINK_STATE_U3) &&
2338                                 (next == DWC3_LINK_STATE_RESUME)) {
2339                         dev_vdbg(dwc->dev, "ignoring transition U3 -> Resume\n");
2340                         return;
2341                 }
2342         }
2343
2344         /*
2345          * WORKAROUND: DWC3 Revisions <1.83a have an issue which, depending
2346          * on the link partner, the USB session might do multiple entry/exit
2347          * of low power states before a transfer takes place.
2348          *
2349          * Due to this problem, we might experience lower throughput. The
2350          * suggested workaround is to disable DCTL[12:9] bits if we're
2351          * transitioning from U1/U2 to U0 and enable those bits again
2352          * after a transfer completes and there are no pending transfers
2353          * on any of the enabled endpoints.
2354          *
2355          * This is the first half of that workaround.
2356          *
2357          * Refers to:
2358          *
2359          * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us
2360          * core send LGO_Ux entering U0
2361          */
2362         if (dwc->revision < DWC3_REVISION_183A) {
2363                 if (next == DWC3_LINK_STATE_U0) {
2364                         u32     u1u2;
2365                         u32     reg;
2366
2367                         switch (dwc->link_state) {
2368                         case DWC3_LINK_STATE_U1:
2369                         case DWC3_LINK_STATE_U2:
2370                                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2371                                 u1u2 = reg & (DWC3_DCTL_INITU2ENA
2372                                                 | DWC3_DCTL_ACCEPTU2ENA
2373                                                 | DWC3_DCTL_INITU1ENA
2374                                                 | DWC3_DCTL_ACCEPTU1ENA);
2375
2376                                 if (!dwc->u1u2)
2377                                         dwc->u1u2 = reg & u1u2;
2378
2379                                 reg &= ~u1u2;
2380
2381                                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2382                                 break;
2383                         default:
2384                                 /* do nothing */
2385                                 break;
2386                         }
2387                 }
2388         }
2389
2390         dwc->link_state = next;
2391
2392         dev_vdbg(dwc->dev, "%s link %d\n", __func__, dwc->link_state);
2393 }
2394
2395 static void dwc3_gadget_interrupt(struct dwc3 *dwc,
2396                 const struct dwc3_event_devt *event)
2397 {
2398         switch (event->type) {
2399         case DWC3_DEVICE_EVENT_DISCONNECT:
2400                 dwc3_gadget_disconnect_interrupt(dwc);
2401                 break;
2402         case DWC3_DEVICE_EVENT_RESET:
2403                 dwc3_gadget_reset_interrupt(dwc);
2404                 break;
2405         case DWC3_DEVICE_EVENT_CONNECT_DONE:
2406                 dwc3_gadget_conndone_interrupt(dwc);
2407                 break;
2408         case DWC3_DEVICE_EVENT_WAKEUP:
2409                 dwc3_gadget_wakeup_interrupt(dwc);
2410                 break;
2411         case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
2412                 dwc3_gadget_linksts_change_interrupt(dwc, event->event_info);
2413                 break;
2414         case DWC3_DEVICE_EVENT_EOPF:
2415                 dev_vdbg(dwc->dev, "End of Periodic Frame\n");
2416                 break;
2417         case DWC3_DEVICE_EVENT_SOF:
2418                 dev_vdbg(dwc->dev, "Start of Periodic Frame\n");
2419                 break;
2420         case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
2421                 dev_vdbg(dwc->dev, "Erratic Error\n");
2422                 break;
2423         case DWC3_DEVICE_EVENT_CMD_CMPL:
2424                 dev_vdbg(dwc->dev, "Command Complete\n");
2425                 break;
2426         case DWC3_DEVICE_EVENT_OVERFLOW:
2427                 dev_vdbg(dwc->dev, "Overflow\n");
2428                 break;
2429         default:
2430                 dev_dbg(dwc->dev, "UNKNOWN IRQ %d\n", event->type);
2431         }
2432 }
2433
2434 static void dwc3_process_event_entry(struct dwc3 *dwc,
2435                 const union dwc3_event *event)
2436 {
2437         /* Endpoint IRQ, handle it and return early */
2438         if (event->type.is_devspec == 0) {
2439                 /* depevt */
2440                 return dwc3_endpoint_interrupt(dwc, &event->depevt);
2441         }
2442
2443         switch (event->type.type) {
2444         case DWC3_EVENT_TYPE_DEV:
2445                 dwc3_gadget_interrupt(dwc, &event->devt);
2446                 break;
2447         /* REVISIT what to do with Carkit and I2C events ? */
2448         default:
2449                 dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw);
2450         }
2451 }
2452
2453 static irqreturn_t dwc3_process_event_buf(struct dwc3 *dwc, u32 buf)
2454 {
2455         struct dwc3_event_buffer *evt;
2456         irqreturn_t ret = IRQ_NONE;
2457         int left;
2458         u32 reg;
2459
2460         evt = dwc->ev_buffs[buf];
2461         left = evt->count;
2462
2463         if (!(evt->flags & DWC3_EVENT_PENDING))
2464                 return IRQ_NONE;
2465
2466         while (left > 0) {
2467                 union dwc3_event event;
2468
2469                 event.raw = *(u32 *) (evt->buf + evt->lpos);
2470
2471                 dwc3_process_event_entry(dwc, &event);
2472
2473                 /*
2474                  * FIXME we wrap around correctly to the next entry as
2475                  * almost all entries are 4 bytes in size. There is one
2476                  * entry which has 12 bytes which is a regular entry
2477                  * followed by 8 bytes data. ATM I don't know how
2478                  * things are organized if we get next to the a
2479                  * boundary so I worry about that once we try to handle
2480                  * that.
2481                  */
2482                 evt->lpos = (evt->lpos + 4) % DWC3_EVENT_BUFFERS_SIZE;
2483                 left -= 4;
2484
2485                 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(buf), 4);
2486         }
2487
2488         evt->count = 0;
2489         evt->flags &= ~DWC3_EVENT_PENDING;
2490         ret = IRQ_HANDLED;
2491
2492         /* Unmask interrupt */
2493         reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(buf));
2494         reg &= ~DWC3_GEVNTSIZ_INTMASK;
2495         dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(buf), reg);
2496
2497         return ret;
2498 }
2499
2500 static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc)
2501 {
2502         struct dwc3 *dwc = _dwc;
2503         unsigned long flags;
2504         irqreturn_t ret = IRQ_NONE;
2505         int i;
2506
2507         spin_lock_irqsave(&dwc->lock, flags);
2508
2509         for (i = 0; i < dwc->num_event_buffers; i++)
2510                 ret |= dwc3_process_event_buf(dwc, i);
2511
2512         spin_unlock_irqrestore(&dwc->lock, flags);
2513
2514         return ret;
2515 }
2516
2517 static irqreturn_t dwc3_check_event_buf(struct dwc3 *dwc, u32 buf)
2518 {
2519         struct dwc3_event_buffer *evt;
2520         u32 count;
2521         u32 reg;
2522
2523         evt = dwc->ev_buffs[buf];
2524
2525         count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(buf));
2526         count &= DWC3_GEVNTCOUNT_MASK;
2527         if (!count)
2528                 return IRQ_NONE;
2529
2530         evt->count = count;
2531         evt->flags |= DWC3_EVENT_PENDING;
2532
2533         /* Mask interrupt */
2534         reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(buf));
2535         reg |= DWC3_GEVNTSIZ_INTMASK;
2536         dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(buf), reg);
2537
2538         return IRQ_WAKE_THREAD;
2539 }
2540
2541 static irqreturn_t dwc3_interrupt(int irq, void *_dwc)
2542 {
2543         struct dwc3                     *dwc = _dwc;
2544         int                             i;
2545         irqreturn_t                     ret = IRQ_NONE;
2546
2547         spin_lock(&dwc->lock);
2548
2549         for (i = 0; i < dwc->num_event_buffers; i++) {
2550                 irqreturn_t status;
2551
2552                 status = dwc3_check_event_buf(dwc, i);
2553                 if (status == IRQ_WAKE_THREAD)
2554                         ret = status;
2555         }
2556
2557         spin_unlock(&dwc->lock);
2558
2559         return ret;
2560 }
2561
2562 /**
2563  * dwc3_gadget_init - Initializes gadget related registers
2564  * @dwc: pointer to our controller context structure
2565  *
2566  * Returns 0 on success otherwise negative errno.
2567  */
2568 int dwc3_gadget_init(struct dwc3 *dwc)
2569 {
2570         int                                     ret;
2571
2572         dwc->ctrl_req = dma_alloc_coherent(dwc->dev, sizeof(*dwc->ctrl_req),
2573                         &dwc->ctrl_req_addr, GFP_KERNEL);
2574         if (!dwc->ctrl_req) {
2575                 dev_err(dwc->dev, "failed to allocate ctrl request\n");
2576                 ret = -ENOMEM;
2577                 goto err0;
2578         }
2579
2580         dwc->ep0_trb = dma_alloc_coherent(dwc->dev, sizeof(*dwc->ep0_trb),
2581                         &dwc->ep0_trb_addr, GFP_KERNEL);
2582         if (!dwc->ep0_trb) {
2583                 dev_err(dwc->dev, "failed to allocate ep0 trb\n");
2584                 ret = -ENOMEM;
2585                 goto err1;
2586         }
2587
2588         dwc->setup_buf = kzalloc(DWC3_EP0_BOUNCE_SIZE, GFP_KERNEL);
2589         if (!dwc->setup_buf) {
2590                 dev_err(dwc->dev, "failed to allocate setup buffer\n");
2591                 ret = -ENOMEM;
2592                 goto err2;
2593         }
2594
2595         dwc->ep0_bounce = dma_alloc_coherent(dwc->dev,
2596                         DWC3_EP0_BOUNCE_SIZE, &dwc->ep0_bounce_addr,
2597                         GFP_KERNEL);
2598         if (!dwc->ep0_bounce) {
2599                 dev_err(dwc->dev, "failed to allocate ep0 bounce buffer\n");
2600                 ret = -ENOMEM;
2601                 goto err3;
2602         }
2603
2604         dwc->gadget.ops                 = &dwc3_gadget_ops;
2605         dwc->gadget.max_speed           = USB_SPEED_SUPER;
2606         dwc->gadget.speed               = USB_SPEED_UNKNOWN;
2607         dwc->gadget.sg_supported        = true;
2608         dwc->gadget.name                = "dwc3-gadget";
2609
2610         /*
2611          * Per databook, DWC3 needs buffer size to be aligned to MaxPacketSize
2612          * on ep out.
2613          */
2614         dwc->gadget.quirk_ep_out_aligned_size = true;
2615
2616         /*
2617          * REVISIT: Here we should clear all pending IRQs to be
2618          * sure we're starting from a well known location.
2619          */
2620
2621         ret = dwc3_gadget_init_endpoints(dwc);
2622         if (ret)
2623                 goto err4;
2624
2625         ret = usb_add_gadget_udc(dwc->dev, &dwc->gadget);
2626         if (ret) {
2627                 dev_err(dwc->dev, "failed to register udc\n");
2628                 goto err4;
2629         }
2630
2631         return 0;
2632
2633 err4:
2634         dwc3_gadget_free_endpoints(dwc);
2635         dma_free_coherent(dwc->dev, DWC3_EP0_BOUNCE_SIZE,
2636                         dwc->ep0_bounce, dwc->ep0_bounce_addr);
2637
2638 err3:
2639         kfree(dwc->setup_buf);
2640
2641 err2:
2642         dma_free_coherent(dwc->dev, sizeof(*dwc->ep0_trb),
2643                         dwc->ep0_trb, dwc->ep0_trb_addr);
2644
2645 err1:
2646         dma_free_coherent(dwc->dev, sizeof(*dwc->ctrl_req),
2647                         dwc->ctrl_req, dwc->ctrl_req_addr);
2648
2649 err0:
2650         return ret;
2651 }
2652
2653 /* -------------------------------------------------------------------------- */
2654
2655 void dwc3_gadget_exit(struct dwc3 *dwc)
2656 {
2657         usb_del_gadget_udc(&dwc->gadget);
2658
2659         dwc3_gadget_free_endpoints(dwc);
2660
2661         dma_free_coherent(dwc->dev, DWC3_EP0_BOUNCE_SIZE,
2662                         dwc->ep0_bounce, dwc->ep0_bounce_addr);
2663
2664         kfree(dwc->setup_buf);
2665
2666         dma_free_coherent(dwc->dev, sizeof(*dwc->ep0_trb),
2667                         dwc->ep0_trb, dwc->ep0_trb_addr);
2668
2669         dma_free_coherent(dwc->dev, sizeof(*dwc->ctrl_req),
2670                         dwc->ctrl_req, dwc->ctrl_req_addr);
2671 }
2672
2673 int dwc3_gadget_prepare(struct dwc3 *dwc)
2674 {
2675         if (dwc->pullups_connected)
2676                 dwc3_gadget_disable_irq(dwc);
2677
2678         return 0;
2679 }
2680
2681 void dwc3_gadget_complete(struct dwc3 *dwc)
2682 {
2683         if (dwc->pullups_connected) {
2684                 dwc3_gadget_enable_irq(dwc);
2685                 dwc3_gadget_run_stop(dwc, true);
2686         }
2687 }
2688
2689 int dwc3_gadget_suspend(struct dwc3 *dwc)
2690 {
2691         __dwc3_gadget_ep_disable(dwc->eps[0]);
2692         __dwc3_gadget_ep_disable(dwc->eps[1]);
2693
2694         dwc->dcfg = dwc3_readl(dwc->regs, DWC3_DCFG);
2695
2696         return 0;
2697 }
2698
2699 int dwc3_gadget_resume(struct dwc3 *dwc)
2700 {
2701         struct dwc3_ep          *dep;
2702         int                     ret;
2703
2704         /* Start with SuperSpeed Default */
2705         dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2706
2707         dep = dwc->eps[0];
2708         ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false);
2709         if (ret)
2710                 goto err0;
2711
2712         dep = dwc->eps[1];
2713         ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false);
2714         if (ret)
2715                 goto err1;
2716
2717         /* begin to receive SETUP packets */
2718         dwc->ep0state = EP0_SETUP_PHASE;
2719         dwc3_ep0_out_start(dwc);
2720
2721         dwc3_writel(dwc->regs, DWC3_DCFG, dwc->dcfg);
2722
2723         return 0;
2724
2725 err1:
2726         __dwc3_gadget_ep_disable(dwc->eps[0]);
2727
2728 err0:
2729         return ret;
2730 }