drivers/usb/cdns3/gadget.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Cadence USBSS DRD Driver - gadget side.
   4  *
   5  * Copyright (C) 2018-2019 Cadence Design Systems.
   6  * Copyright (C) 2017-2018 NXP
   7  *
   8  * Authors: Pawel Jez <pjez@cadence.com>,
   9  *          Pawel Laszczak <pawell@cadence.com>
  10  *          Peter Chen <peter.chen@nxp.com>
  11  */
  12
  13 /*
  14  * Work around 1:
  15  * At some situations, the controller may get stale data address in TRB
  16  * at below sequences:
  17  * 1. Controller read TRB includes data address
  18  * 2. Software updates TRBs includes data address and Cycle bit
  19  * 3. Controller read TRB which includes Cycle bit
  20  * 4. DMA run with stale data address
  21  *
  22  * To fix this problem, driver needs to make the first TRB in TD as invalid.
  23  * After preparing all TRBs driver needs to check the position of DMA and
  24  * if the DMA point to the first just added TRB and doorbell is 1,
  25  * then driver must defer making this TRB as valid. This TRB will be make
  26  * as valid during adding next TRB only if DMA is stopped or at TRBERR
  27  * interrupt.
  28  *
  29  * Issue has been fixed in DEV_VER_V3 version of controller.
  30  *
  31  */
  32
  33 #include <linux/dma-mapping.h>
  34 #include <linux/usb/gadget.h>
  35 #include <linux/module.h>
  36 #include <linux/iopoll.h>
  37
  38 #include "core.h"
  39 #include "gadget-export.h"
  40 #include "gadget.h"
  41 #include "trace.h"
  42 #include "drd.h"
  43
  44 static int __cdns3_gadget_ep_queue(struct usb_ep *ep,
  45                                    struct usb_request *request,
  46                                    gfp_t gfp_flags);
  47
  48 /**
  49  * cdns3_set_register_bit - set bit in given register.
  50  * @ptr: address of device controller register to be read and changed
  51  * @mask: bits requested to set
  52  */
  53 void cdns3_set_register_bit(void __iomem *ptr, u32 mask)
  54 {
  55         mask = readl(ptr) | mask;
  56         writel(mask, ptr);
  57 }
  58
  59 /**
  60  * cdns3_ep_addr_to_index - Macro converts endpoint address to
  61  * index of endpoint object in cdns3_device.eps[] container
  62  * @ep_addr: endpoint address for which endpoint object is required
  63  *
  64  */
  65 u8 cdns3_ep_addr_to_index(u8 ep_addr)
  66 {
  67         return (((ep_addr & 0x7F)) + ((ep_addr & USB_DIR_IN) ? 16 : 0));
  68 }
  69
  70 static int cdns3_get_dma_pos(struct cdns3_device *priv_dev,
  71                              struct cdns3_endpoint *priv_ep)
  72 {
  73         int dma_index;
  74
  75         dma_index = readl(&priv_dev->regs->ep_traddr) - priv_ep->trb_pool_dma;
  76
  77         return dma_index / TRB_SIZE;
  78 }
  79
  80 /**
  81  * cdns3_next_request - returns next request from list
  82  * @list: list containing requests
  83  *
  84  * Returns request or NULL if no requests in list
  85  */
  86 struct usb_request *cdns3_next_request(struct list_head *list)
  87 {
  88         return list_first_entry_or_null(list, struct usb_request, list);
  89 }
  90
  91 /**
  92  * cdns3_next_align_buf - returns next buffer from list
  93  * @list: list containing buffers
  94  *
  95  * Returns buffer or NULL if no buffers in list
  96  */
  97 struct cdns3_aligned_buf *cdns3_next_align_buf(struct list_head *list)
  98 {
  99         return list_first_entry_or_null(list, struct cdns3_aligned_buf, list);
 100 }
 101
 102 /**
 103  * select_ep - selects endpoint
 104  * @priv_dev:  extended gadget object
 105  * @ep: endpoint address
 106  */
 107 void cdns3_select_ep(struct cdns3_device *priv_dev, u32 ep)
 108 {
 109         if (priv_dev->selected_ep == ep)
 110                 return;
 111
 112         priv_dev->selected_ep = ep;
 113         writel(ep, &priv_dev->regs->ep_sel);
 114 }
 115
 116 dma_addr_t cdns3_trb_virt_to_dma(struct cdns3_endpoint *priv_ep,
 117                                  struct cdns3_trb *trb)
 118 {
 119         u32 offset = (char *)trb - (char *)priv_ep->trb_pool;
 120
 121         return priv_ep->trb_pool_dma + offset;
 122 }
 123
 124 int cdns3_ring_size(struct cdns3_endpoint *priv_ep)
 125 {
 126         switch (priv_ep->type) {
 127         case USB_ENDPOINT_XFER_ISOC:
 128                 return TRB_ISO_RING_SIZE;
 129         case USB_ENDPOINT_XFER_CONTROL:
 130                 return TRB_CTRL_RING_SIZE;
 131         default:
 132                 return TRB_RING_SIZE;
 133         }
 134 }
 135
 136 /**
 137  * cdns3_allocate_trb_pool - Allocates TRB's pool for selected endpoint
 138  * @priv_ep:  endpoint object
 139  *
 140  * Function will return 0 on success or -ENOMEM on allocation error
 141  */
 142 int cdns3_allocate_trb_pool(struct cdns3_endpoint *priv_ep)
 143 {
 144         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 145         int ring_size = cdns3_ring_size(priv_ep);
 146         struct cdns3_trb *link_trb;
 147
 148         if (!priv_ep->trb_pool) {
 149                 priv_ep->trb_pool = dma_alloc_coherent(priv_dev->sysdev,
 150                                                        ring_size,
 151                                                        &priv_ep->trb_pool_dma,
 152                                                        GFP_DMA32 | GFP_ATOMIC);
 153                 if (!priv_ep->trb_pool)
 154                         return -ENOMEM;
 155         } else {
 156                 memset(priv_ep->trb_pool, 0, ring_size);
 157         }
 158
 159         if (!priv_ep->num)
 160                 return 0;
 161
 162         priv_ep->num_trbs = ring_size / TRB_SIZE;
 163         /* Initialize the last TRB as Link TRB. */
 164         link_trb = (priv_ep->trb_pool + (priv_ep->num_trbs - 1));
 165         link_trb->buffer = TRB_BUFFER(priv_ep->trb_pool_dma);
 166         link_trb->control = TRB_CYCLE | TRB_TYPE(TRB_LINK) | TRB_TOGGLE;
 167
 168         return 0;
 169 }
 170
 171 static void cdns3_free_trb_pool(struct cdns3_endpoint *priv_ep)
 172 {
 173         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 174
 175         if (priv_ep->trb_pool) {
 176                 dma_free_coherent(priv_dev->sysdev,
 177                                   cdns3_ring_size(priv_ep),
 178                                   priv_ep->trb_pool, priv_ep->trb_pool_dma);
 179                 priv_ep->trb_pool = NULL;
 180         }
 181 }
 182
 183 /**
 184  * cdns3_ep_stall_flush - Stalls and flushes selected endpoint
 185  * @priv_ep: endpoint object
 186  *
 187  * Endpoint must be selected before call to this function
 188  */
 189 static void cdns3_ep_stall_flush(struct cdns3_endpoint *priv_ep)
 190 {
 191         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 192         int val;
 193
 194         trace_cdns3_halt(priv_ep, 1, 1);
 195
 196         writel(EP_CMD_DFLUSH | EP_CMD_ERDY | EP_CMD_SSTALL,
 197                &priv_dev->regs->ep_cmd);
 198
 199         /* wait for DFLUSH cleared */
 200         readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
 201                                   !(val & EP_CMD_DFLUSH), 1, 1000);
 202         priv_ep->flags |= EP_STALLED;
 203         priv_ep->flags &= ~EP_STALL_PENDING;
 204 }
 205
 206 /**
 207  * cdns3_hw_reset_eps_config - reset endpoints configuration kept by controller.
 208  * @priv_dev: extended gadget object
 209  */
 210 void cdns3_hw_reset_eps_config(struct cdns3_device *priv_dev)
 211 {
 212         writel(USB_CONF_CFGRST, &priv_dev->regs->usb_conf);
 213
 214         cdns3_allow_enable_l1(priv_dev, 0);
 215         priv_dev->hw_configured_flag = 0;
 216         priv_dev->onchip_used_size = 0;
 217         priv_dev->out_mem_is_allocated = 0;
 218         priv_dev->wait_for_setup = 0;
 219 }
 220
 221 /**
 222  * cdns3_ep_inc_trb - increment a trb index.
 223  * @index: Pointer to the TRB index to increment.
 224  * @cs: Cycle state
 225  * @trb_in_seg: number of TRBs in segment
 226  *
 227  * The index should never point to the link TRB. After incrementing,
 228  * if it is point to the link TRB, wrap around to the beginning and revert
 229  * cycle state bit The
 230  * link TRB is always at the last TRB entry.
 231  */
 232 static void cdns3_ep_inc_trb(int *index, u8 *cs, int trb_in_seg)
 233 {
 234         (*index)++;
 235         if (*index == (trb_in_seg - 1)) {
 236                 *index = 0;
 237                 *cs ^=  1;
 238         }
 239 }
 240
 241 /**
 242  * cdns3_ep_inc_enq - increment endpoint's enqueue pointer
 243  * @priv_ep: The endpoint whose enqueue pointer we're incrementing
 244  */
 245 static void cdns3_ep_inc_enq(struct cdns3_endpoint *priv_ep)
 246 {
 247         priv_ep->free_trbs--;
 248         cdns3_ep_inc_trb(&priv_ep->enqueue, &priv_ep->pcs, priv_ep->num_trbs);
 249 }
 250
 251 /**
 252  * cdns3_ep_inc_deq - increment endpoint's dequeue pointer
 253  * @priv_ep: The endpoint whose dequeue pointer we're incrementing
 254  */
 255 static void cdns3_ep_inc_deq(struct cdns3_endpoint *priv_ep)
 256 {
 257         priv_ep->free_trbs++;
 258         cdns3_ep_inc_trb(&priv_ep->dequeue, &priv_ep->ccs, priv_ep->num_trbs);
 259 }
 260
 261 void cdns3_move_deq_to_next_trb(struct cdns3_request *priv_req)
 262 {
 263         struct cdns3_endpoint *priv_ep = priv_req->priv_ep;
 264         int current_trb = priv_req->start_trb;
 265
 266         while (current_trb != priv_req->end_trb) {
 267                 cdns3_ep_inc_deq(priv_ep);
 268                 current_trb = priv_ep->dequeue;
 269         }
 270
 271         cdns3_ep_inc_deq(priv_ep);
 272 }
 273
 274 /**
 275  * cdns3_allow_enable_l1 - enable/disable permits to transition to L1.
 276  * @priv_dev: Extended gadget object
 277  * @enable: Enable/disable permit to transition to L1.
 278  *
 279  * If bit USB_CONF_L1EN is set and device receive Extended Token packet,
 280  * then controller answer with ACK handshake.
 281  * If bit USB_CONF_L1DS is set and device receive Extended Token packet,
 282  * then controller answer with NYET handshake.
 283  */
 284 void cdns3_allow_enable_l1(struct cdns3_device *priv_dev, int enable)
 285 {
 286         if (enable)
 287                 writel(USB_CONF_L1EN, &priv_dev->regs->usb_conf);
 288         else
 289                 writel(USB_CONF_L1DS, &priv_dev->regs->usb_conf);
 290 }
 291
 292 enum usb_device_speed cdns3_get_speed(struct cdns3_device *priv_dev)
 293 {
 294         u32 reg;
 295
 296         reg = readl(&priv_dev->regs->usb_sts);
 297
 298         if (DEV_SUPERSPEED(reg))
 299                 return USB_SPEED_SUPER;
 300         else if (DEV_HIGHSPEED(reg))
 301                 return USB_SPEED_HIGH;
 302         else if (DEV_FULLSPEED(reg))
 303                 return USB_SPEED_FULL;
 304         else if (DEV_LOWSPEED(reg))
 305                 return USB_SPEED_LOW;
 306         return USB_SPEED_UNKNOWN;
 307 }
 308
 309 /**
 310  * cdns3_start_all_request - add to ring all request not started
 311  * @priv_dev: Extended gadget object
 312  * @priv_ep: The endpoint for whom request will be started.
 313  *
 314  * Returns return ENOMEM if transfer ring i not enough TRBs to start
 315  *         all requests.
 316  */
 317 static int cdns3_start_all_request(struct cdns3_device *priv_dev,
 318                                    struct cdns3_endpoint *priv_ep)
 319 {
 320         struct cdns3_request *priv_req;
 321         struct usb_request *request;
 322         int ret = 0;
 323
 324         while (!list_empty(&priv_ep->deferred_req_list)) {
 325                 request = cdns3_next_request(&priv_ep->deferred_req_list);
 326                 priv_req = to_cdns3_request(request);
 327
 328                 ret = cdns3_ep_run_transfer(priv_ep, request);
 329                 if (ret)
 330                         return ret;
 331
 332                 list_del(&request->list);
 333                 list_add_tail(&request->list,
 334                               &priv_ep->pending_req_list);
 335         }
 336
 337         priv_ep->flags &= ~EP_RING_FULL;
 338         return ret;
 339 }
 340
 341 /**
 342  * cdns3_gadget_giveback - call struct usb_request's ->complete callback
 343  * @priv_ep: The endpoint to whom the request belongs to
 344  * @priv_req: The request we're giving back
 345  * @status: completion code for the request
 346  *
 347  * Must be called with controller's lock held and interrupts disabled. This
 348  * function will unmap @req and call its ->complete() callback to notify upper
 349  * layers that it has completed.
 350  */
 351 void cdns3_gadget_giveback(struct cdns3_endpoint *priv_ep,
 352                            struct cdns3_request *priv_req,
 353                            int status)
 354 {
 355         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 356         struct usb_request *request = &priv_req->request;
 357
 358         list_del_init(&request->list);
 359
 360         if (request->status == -EINPROGRESS)
 361                 request->status = status;
 362
 363         usb_gadget_unmap_request_by_dev(priv_dev->sysdev, request,
 364                                         priv_ep->dir);
 365
 366         if ((priv_req->flags & REQUEST_UNALIGNED) &&
 367             priv_ep->dir == USB_DIR_OUT && !request->status)
 368                 memcpy(request->buf, priv_req->aligned_buf->buf,
 369                        request->length);
 370
 371         priv_req->flags &= ~(REQUEST_PENDING | REQUEST_UNALIGNED);
 372         trace_cdns3_gadget_giveback(priv_req);
 373
 374         if (request->complete) {
 375                 spin_unlock(&priv_dev->lock);
 376                 usb_gadget_giveback_request(&priv_ep->endpoint,
 377                                             request);
 378                 spin_lock(&priv_dev->lock);
 379         }
 380
 381         if (request->buf == priv_dev->zlp_buf)
 382                 cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
 383 }
 384
 385 void cdns3_wa1_restore_cycle_bit(struct cdns3_endpoint *priv_ep)
 386 {
 387         /* Work around for stale data address in TRB*/
 388         if (priv_ep->wa1_set) {
 389                 trace_cdns3_wa1(priv_ep, "restore cycle bit");
 390
 391                 priv_ep->wa1_set = 0;
 392                 priv_ep->wa1_trb_index = 0xFFFF;
 393                 if (priv_ep->wa1_cycle_bit) {
 394                         priv_ep->wa1_trb->control =
 395                                 priv_ep->wa1_trb->control | 0x1;
 396                 } else {
 397                         priv_ep->wa1_trb->control =
 398                                 priv_ep->wa1_trb->control & ~0x1;
 399                 }
 400         }
 401 }
 402
 403 static void cdns3_free_aligned_request_buf(struct work_struct *work)
 404 {
 405         struct cdns3_device *priv_dev = container_of(work, struct cdns3_device,
 406                                         aligned_buf_wq);
 407         struct cdns3_aligned_buf *buf, *tmp;
 408         unsigned long flags;
 409
 410         spin_lock_irqsave(&priv_dev->lock, flags);
 411
 412         list_for_each_entry_safe(buf, tmp, &priv_dev->aligned_buf_list, list) {
 413                 if (!buf->in_use) {
 414                         list_del(&buf->list);
 415
 416                         /*
 417                          * Re-enable interrupts to free DMA capable memory.
 418                          * Driver can't free this memory with disabled
 419                          * interrupts.
 420                          */
 421                         spin_unlock_irqrestore(&priv_dev->lock, flags);
 422                         dma_free_coherent(priv_dev->sysdev, buf->size,
 423                                           buf->buf, buf->dma);
 424                         kfree(buf);
 425                         spin_lock_irqsave(&priv_dev->lock, flags);
 426                 }
 427         }
 428
 429         spin_unlock_irqrestore(&priv_dev->lock, flags);
 430 }
 431
 432 static int cdns3_prepare_aligned_request_buf(struct cdns3_request *priv_req)
 433 {
 434         struct cdns3_endpoint *priv_ep = priv_req->priv_ep;
 435         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 436         struct cdns3_aligned_buf *buf;
 437
 438         /* check if buffer is aligned to 8. */
 439         if (!((uintptr_t)priv_req->request.buf & 0x7))
 440                 return 0;
 441
 442         buf = priv_req->aligned_buf;
 443
 444         if (!buf || priv_req->request.length > buf->size) {
 445                 buf = kzalloc(sizeof(*buf), GFP_ATOMIC);
 446                 if (!buf)
 447                         return -ENOMEM;
 448
 449                 buf->size = priv_req->request.length;
 450
 451                 buf->buf = dma_alloc_coherent(priv_dev->sysdev,
 452                                               buf->size,
 453                                               &buf->dma,
 454                                               GFP_ATOMIC);
 455                 if (!buf->buf) {
 456                         kfree(buf);
 457                         return -ENOMEM;
 458                 }
 459
 460                 if (priv_req->aligned_buf) {
 461                         trace_cdns3_free_aligned_request(priv_req);
 462                         priv_req->aligned_buf->in_use = 0;
 463                         queue_work(system_freezable_wq,
 464                                    &priv_dev->aligned_buf_wq);
 465                 }
 466
 467                 buf->in_use = 1;
 468                 priv_req->aligned_buf = buf;
 469
 470                 list_add_tail(&buf->list,
 471                               &priv_dev->aligned_buf_list);
 472         }
 473
 474         if (priv_ep->dir == USB_DIR_IN) {
 475                 memcpy(buf->buf, priv_req->request.buf,
 476                        priv_req->request.length);
 477         }
 478
 479         priv_req->flags |= REQUEST_UNALIGNED;
 480         trace_cdns3_prepare_aligned_request(priv_req);
 481
 482         return 0;
 483 }
 484
 485 static int cdns3_wa1_update_guard(struct cdns3_endpoint *priv_ep,
 486                                   struct cdns3_trb *trb)
 487 {
 488         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 489
 490         if (!priv_ep->wa1_set) {
 491                 u32 doorbell;
 492
 493                 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
 494
 495                 if (doorbell) {
 496                         priv_ep->wa1_cycle_bit = priv_ep->pcs ? TRB_CYCLE : 0;
 497                         priv_ep->wa1_set = 1;
 498                         priv_ep->wa1_trb = trb;
 499                         priv_ep->wa1_trb_index = priv_ep->enqueue;
 500                         trace_cdns3_wa1(priv_ep, "set guard");
 501                         return 0;
 502                 }
 503         }
 504         return 1;
 505 }
 506
 507 static void cdns3_wa1_tray_restore_cycle_bit(struct cdns3_device *priv_dev,
 508                                              struct cdns3_endpoint *priv_ep)
 509 {
 510         int dma_index;
 511         u32 doorbell;
 512
 513         doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
 514         dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
 515
 516         if (!doorbell || dma_index != priv_ep->wa1_trb_index)
 517                 cdns3_wa1_restore_cycle_bit(priv_ep);
 518 }
 519
 520 /**
 521  * cdns3_ep_run_transfer - start transfer on no-default endpoint hardware
 522  * @priv_ep: endpoint object
 523  *
 524  * Returns zero on success or negative value on failure
 525  */
 526 int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
 527                           struct usb_request *request)
 528 {
 529         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 530         struct cdns3_request *priv_req;
 531         struct cdns3_trb *trb;
 532         dma_addr_t trb_dma;
 533         u32 togle_pcs = 1;
 534         int sg_iter = 0;
 535         int num_trb;
 536         int address;
 537         u32 control;
 538         int pcs;
 539
 540         if (priv_ep->type == USB_ENDPOINT_XFER_ISOC)
 541                 num_trb = priv_ep->interval;
 542         else
 543                 num_trb = request->num_sgs ? request->num_sgs : 1;
 544
 545         if (num_trb > priv_ep->free_trbs) {
 546                 priv_ep->flags |= EP_RING_FULL;
 547                 return -ENOBUFS;
 548         }
 549
 550         priv_req = to_cdns3_request(request);
 551         address = priv_ep->endpoint.desc->bEndpointAddress;
 552
 553         priv_ep->flags |= EP_PENDING_REQUEST;
 554
 555         /* must allocate buffer aligned to 8 */
 556         if (priv_req->flags & REQUEST_UNALIGNED)
 557                 trb_dma = priv_req->aligned_buf->dma;
 558         else
 559                 trb_dma = request->dma;
 560
 561         trb = priv_ep->trb_pool + priv_ep->enqueue;
 562         priv_req->start_trb = priv_ep->enqueue;
 563         priv_req->trb = trb;
 564
 565         cdns3_select_ep(priv_ep->cdns3_dev, address);
 566
 567         /* prepare ring */
 568         if ((priv_ep->enqueue + num_trb)  >= (priv_ep->num_trbs - 1)) {
 569                 struct cdns3_trb *link_trb;
 570                 int doorbell, dma_index;
 571                 u32 ch_bit = 0;
 572
 573                 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
 574                 dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
 575
 576                 /* Driver can't update LINK TRB if it is current processed. */
 577                 if (doorbell && dma_index == priv_ep->num_trbs - 1) {
 578                         priv_ep->flags |= EP_DEFERRED_DRDY;
 579                         return -ENOBUFS;
 580                 }
 581
 582                 /*updating C bt in  Link TRB before starting DMA*/
 583                 link_trb = priv_ep->trb_pool + (priv_ep->num_trbs - 1);
 584                 /*
 585                  * For TRs size equal 2 enabling TRB_CHAIN for epXin causes
 586                  * that DMA stuck at the LINK TRB.
 587                  * On the other hand, removing TRB_CHAIN for longer TRs for
 588                  * epXout cause that DMA stuck after handling LINK TRB.
 589                  * To eliminate this strange behavioral driver set TRB_CHAIN
 590                  * bit only for TR size > 2.
 591                  */
 592                 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC ||
 593                     TRBS_PER_SEGMENT > 2)
 594                         ch_bit = TRB_CHAIN;
 595
 596                 link_trb->control = ((priv_ep->pcs) ? TRB_CYCLE : 0) |
 597                                     TRB_TYPE(TRB_LINK) | TRB_TOGGLE | ch_bit;
 598         }
 599
 600         if (priv_dev->dev_ver <= DEV_VER_V2)
 601                 togle_pcs = cdns3_wa1_update_guard(priv_ep, trb);
 602
 603         /* set incorrect Cycle Bit for first trb*/
 604         control = priv_ep->pcs ? 0 : TRB_CYCLE;
 605
 606         do {
 607                 u32 length;
 608                 u16 td_size = 0;
 609
 610                 /* fill TRB */
 611                 control |= TRB_TYPE(TRB_NORMAL);
 612                 trb->buffer = TRB_BUFFER(request->num_sgs == 0
 613                                 ? trb_dma : request->sg[sg_iter].dma_address);
 614
 615                 if (likely(!request->num_sgs))
 616                         length = request->length;
 617                 else
 618                         length = request->sg[sg_iter].length;
 619
 620                 if (likely(priv_dev->dev_ver >= DEV_VER_V2))
 621                         td_size = DIV_ROUND_UP(length,
 622                                                priv_ep->endpoint.maxpacket);
 623
 624                 trb->length = TRB_BURST_LEN(priv_ep->trb_burst_size) |
 625                                         TRB_LEN(length);
 626                 if (priv_dev->gadget.speed == USB_SPEED_SUPER)
 627                         trb->length |= TRB_TDL_SS_SIZE(td_size);
 628                 else
 629                         control |= TRB_TDL_HS_SIZE(td_size);
 630
 631                 pcs = priv_ep->pcs ? TRB_CYCLE : 0;
 632
 633                 /*
 634                  * first trb should be prepared as last to avoid processing
 635                  *  transfer to early
 636                  */
 637                 if (sg_iter != 0)
 638                         control |= pcs;
 639
 640                 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC  && !priv_ep->dir) {
 641                         control |= TRB_IOC | TRB_ISP;
 642                 } else {
 643                         /* for last element in TD or in SG list */
 644                         if (sg_iter == (num_trb - 1) && sg_iter != 0)
 645                                 control |= pcs | TRB_IOC | TRB_ISP;
 646                 }
 647
 648                 if (sg_iter)
 649                         trb->control = control;
 650                 else
 651                         priv_req->trb->control = control;
 652
 653                 control = 0;
 654                 ++sg_iter;
 655                 priv_req->end_trb = priv_ep->enqueue;
 656                 cdns3_ep_inc_enq(priv_ep);
 657                 trb = priv_ep->trb_pool + priv_ep->enqueue;
 658         } while (sg_iter < num_trb);
 659
 660         trb = priv_req->trb;
 661
 662         priv_req->flags |= REQUEST_PENDING;
 663
 664         if (sg_iter == 1)
 665                 trb->control |= TRB_IOC | TRB_ISP;
 666
 667         /*
 668          * Memory barrier - cycle bit must be set before other filds in trb.
 669          */
 670         wmb();
 671
 672         /* give the TD to the consumer*/
 673         if (togle_pcs)
 674                 trb->control =  trb->control ^ 1;
 675
 676         if (priv_dev->dev_ver <= DEV_VER_V2)
 677                 cdns3_wa1_tray_restore_cycle_bit(priv_dev, priv_ep);
 678
 679         trace_cdns3_prepare_trb(priv_ep, priv_req->trb);
 680
 681         /*
 682          * Memory barrier - Cycle Bit must be set before trb->length  and
 683          * trb->buffer fields.
 684          */
 685         wmb();
 686
 687         /*
 688          * For DMULT mode we can set address to transfer ring only once after
 689          * enabling endpoint.
 690          */
 691         if (priv_ep->flags & EP_UPDATE_EP_TRBADDR) {
 692                 /*
 693                  * Until SW is not ready to handle the OUT transfer the ISO OUT
 694                  * Endpoint should be disabled (EP_CFG.ENABLE = 0).
 695                  * EP_CFG_ENABLE must be set before updating ep_traddr.
 696                  */
 697                 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC  && !priv_ep->dir &&
 698                     !(priv_ep->flags & EP_QUIRK_ISO_OUT_EN)) {
 699                         priv_ep->flags |= EP_QUIRK_ISO_OUT_EN;
 700                         cdns3_set_register_bit(&priv_dev->regs->ep_cfg,
 701                                                EP_CFG_ENABLE);
 702                 }
 703
 704                 writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma +
 705                                         priv_req->start_trb * TRB_SIZE),
 706                                         &priv_dev->regs->ep_traddr);
 707
 708                 priv_ep->flags &= ~EP_UPDATE_EP_TRBADDR;
 709         }
 710
 711         if (!priv_ep->wa1_set && !(priv_ep->flags & EP_STALLED)) {
 712                 trace_cdns3_ring(priv_ep);
 713                 /*clearing TRBERR and EP_STS_DESCMIS before seting DRDY*/
 714                 writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts);
 715                 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
 716                 trace_cdns3_doorbell_epx(priv_ep->name,
 717                                          readl(&priv_dev->regs->ep_traddr));
 718         }
 719
 720         /* WORKAROUND for transition to L0 */
 721         __cdns3_gadget_wakeup(priv_dev);
 722
 723         return 0;
 724 }
 725
 726 void cdns3_set_hw_configuration(struct cdns3_device *priv_dev)
 727 {
 728         struct cdns3_endpoint *priv_ep;
 729         struct usb_ep *ep;
 730         int val;
 731
 732         if (priv_dev->hw_configured_flag)
 733                 return;
 734
 735         writel(USB_CONF_CFGSET, &priv_dev->regs->usb_conf);
 736         writel(EP_CMD_ERDY | EP_CMD_REQ_CMPL, &priv_dev->regs->ep_cmd);
 737
 738         cdns3_set_register_bit(&priv_dev->regs->usb_conf,
 739                                USB_CONF_U1EN | USB_CONF_U2EN);
 740
 741         /* wait until configuration set */
 742         readl_poll_timeout_atomic(&priv_dev->regs->usb_sts, val,
 743                                   val & USB_STS_CFGSTS_MASK, 1, 100);
 744
 745         priv_dev->hw_configured_flag = 1;
 746
 747         list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
 748                 if (ep->enabled) {
 749                         priv_ep = ep_to_cdns3_ep(ep);
 750                         cdns3_start_all_request(priv_dev, priv_ep);
 751                 }
 752         }
 753 }
 754
 755 /**
 756  * cdns3_request_handled - check whether request has been handled by DMA
 757  *
 758  * @priv_ep: extended endpoint object.
 759  * @priv_req: request object for checking
 760  *
 761  * Endpoint must be selected before invoking this function.
 762  *
 763  * Returns false if request has not been handled by DMA, else returns true.
 764  *
 765  * SR - start ring
 766  * ER -  end ring
 767  * DQ = priv_ep->dequeue - dequeue position
 768  * EQ = priv_ep->enqueue -  enqueue position
 769  * ST = priv_req->start_trb - index of first TRB in transfer ring
 770  * ET = priv_req->end_trb - index of last TRB in transfer ring
 771  * CI = current_index - index of processed TRB by DMA.
 772  *
 773  * As first step, function checks if cycle bit for priv_req->start_trb is
 774  * correct.
 775  *
 776  * some rules:
 777  * 1. priv_ep->dequeue never exceed current_index.
 778  * 2  priv_ep->enqueue never exceed priv_ep->dequeue
 779  * 3. exception: priv_ep->enqueue == priv_ep->dequeue
 780  *    and priv_ep->free_trbs is zero.
 781  *    This case indicate that TR is full.
 782  *
 783  * Then We can split recognition into two parts:
 784  * Case 1 - priv_ep->dequeue < current_index
 785  *      SR ... EQ ... DQ ... CI ... ER
 786  *      SR ... DQ ... CI ... EQ ... ER
 787  *
 788  *      Request has been handled by DMA if ST and ET is between DQ and CI.
 789  *
 790  * Case 2 - priv_ep->dequeue > current_index
 791  * This situation take place when CI go through the LINK TRB at the end of
 792  * transfer ring.
 793  *      SR ... CI ... EQ ... DQ ... ER
 794  *
 795  *      Request has been handled by DMA if ET is less then CI or
 796  *      ET is greater or equal DQ.
 797  */
 798 static bool cdns3_request_handled(struct cdns3_endpoint *priv_ep,
 799                                   struct cdns3_request *priv_req)
 800 {
 801         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 802         struct cdns3_trb *trb = priv_req->trb;
 803         int current_index = 0;
 804         int handled = 0;
 805         int doorbell;
 806
 807         current_index = cdns3_get_dma_pos(priv_dev, priv_ep);
 808         doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
 809
 810         trb = &priv_ep->trb_pool[priv_req->start_trb];
 811
 812         if ((trb->control  & TRB_CYCLE) != priv_ep->ccs)
 813                 goto finish;
 814
 815         if (doorbell == 1 && current_index == priv_ep->dequeue)
 816                 goto finish;
 817
 818         /* The corner case for TRBS_PER_SEGMENT equal 2). */
 819         if (TRBS_PER_SEGMENT == 2 && priv_ep->type != USB_ENDPOINT_XFER_ISOC) {
 820                 handled = 1;
 821                 goto finish;
 822         }
 823
 824         if (priv_ep->enqueue == priv_ep->dequeue &&
 825             priv_ep->free_trbs == 0) {
 826                 handled = 1;
 827         } else if (priv_ep->dequeue < current_index) {
 828                 if ((current_index == (priv_ep->num_trbs - 1)) &&
 829                     !priv_ep->dequeue)
 830                         goto finish;
 831
 832                 if (priv_req->end_trb >= priv_ep->dequeue &&
 833                     priv_req->end_trb < current_index)
 834                         handled = 1;
 835         } else if (priv_ep->dequeue  > current_index) {
 836                 if (priv_req->end_trb  < current_index ||
 837                     priv_req->end_trb >= priv_ep->dequeue)
 838                         handled = 1;
 839         }
 840
 841 finish:
 842         trace_cdns3_request_handled(priv_req, current_index, handled);
 843
 844         return handled;
 845 }
 846
 847 static void cdns3_transfer_completed(struct cdns3_device *priv_dev,
 848                                      struct cdns3_endpoint *priv_ep)
 849 {
 850         struct cdns3_request *priv_req;
 851         struct usb_request *request;
 852         struct cdns3_trb *trb;
 853
 854         while (!list_empty(&priv_ep->pending_req_list)) {
 855                 request = cdns3_next_request(&priv_ep->pending_req_list);
 856                 priv_req = to_cdns3_request(request);
 857
 858                 /* Re-select endpoint. It could be changed by other CPU during
 859                  * handling usb_gadget_giveback_request.
 860                  */
 861                 cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
 862
 863                 if (!cdns3_request_handled(priv_ep, priv_req))
 864                         goto prepare_next_td;
 865
 866                 trb = priv_ep->trb_pool + priv_ep->dequeue;
 867                 trace_cdns3_complete_trb(priv_ep, trb);
 868
 869                 if (trb != priv_req->trb)
 870                         dev_warn(priv_dev->dev,
 871                                  "request_trb=0x%p, queue_trb=0x%p\n",
 872                                  priv_req->trb, trb);
 873
 874                 request->actual = TRB_LEN(le32_to_cpu(trb->length));
 875                 cdns3_move_deq_to_next_trb(priv_req);
 876                 cdns3_gadget_giveback(priv_ep, priv_req, 0);
 877
 878                 if (priv_ep->type != USB_ENDPOINT_XFER_ISOC &&
 879                     TRBS_PER_SEGMENT == 2)
 880                         break;
 881         }
 882         priv_ep->flags &= ~EP_PENDING_REQUEST;
 883
 884 prepare_next_td:
 885         if (!(priv_ep->flags & EP_STALLED) &&
 886             !(priv_ep->flags & EP_STALL_PENDING))
 887                 cdns3_start_all_request(priv_dev, priv_ep);
 888 }
 889
 890 void cdns3_rearm_transfer(struct cdns3_endpoint *priv_ep, u8 rearm)
 891 {
 892         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 893
 894         cdns3_wa1_restore_cycle_bit(priv_ep);
 895
 896         if (rearm) {
 897                 trace_cdns3_ring(priv_ep);
 898
 899                 /* Cycle Bit must be updated before arming DMA. */
 900                 wmb();
 901                 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
 902
 903                 __cdns3_gadget_wakeup(priv_dev);
 904
 905                 trace_cdns3_doorbell_epx(priv_ep->name,
 906                                          readl(&priv_dev->regs->ep_traddr));
 907         }
 908 }
 909
 910 /**
 911  * cdns3_check_ep_interrupt_proceed - Processes interrupt related to endpoint
 912  * @priv_ep: endpoint object
 913  *
 914  * Returns 0
 915  */
 916 static int cdns3_check_ep_interrupt_proceed(struct cdns3_endpoint *priv_ep)
 917 {
 918         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 919         u32 ep_sts_reg;
 920
 921         cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
 922
 923         trace_cdns3_epx_irq(priv_dev, priv_ep);
 924
 925         ep_sts_reg = readl(&priv_dev->regs->ep_sts);
 926         writel(ep_sts_reg, &priv_dev->regs->ep_sts);
 927
 928         if (ep_sts_reg & EP_STS_TRBERR) {
 929                 if (priv_ep->flags & EP_STALL_PENDING &&
 930                     !(ep_sts_reg & EP_STS_DESCMIS &&
 931                     priv_dev->dev_ver < DEV_VER_V2)) {
 932                         cdns3_ep_stall_flush(priv_ep);
 933                 }
 934
 935                 /*
 936                  * For isochronous transfer driver completes request on
 937                  * IOC or on TRBERR. IOC appears only when device receive
 938                  * OUT data packet. If host disable stream or lost some packet
 939                  * then the only way to finish all queued transfer is to do it
 940                  * on TRBERR event.
 941                  */
 942                 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC &&
 943                     !priv_ep->wa1_set) {
 944                         if (!priv_ep->dir) {
 945                                 u32 ep_cfg = readl(&priv_dev->regs->ep_cfg);
 946
 947                                 ep_cfg &= ~EP_CFG_ENABLE;
 948                                 writel(ep_cfg, &priv_dev->regs->ep_cfg);
 949                                 priv_ep->flags &= ~EP_QUIRK_ISO_OUT_EN;
 950                         }
 951                         cdns3_transfer_completed(priv_dev, priv_ep);
 952                 } else if (!(priv_ep->flags & EP_STALLED) &&
 953                           !(priv_ep->flags & EP_STALL_PENDING)) {
 954                         if (priv_ep->flags & EP_DEFERRED_DRDY) {
 955                                 priv_ep->flags &= ~EP_DEFERRED_DRDY;
 956                                 cdns3_start_all_request(priv_dev, priv_ep);
 957                         } else {
 958                                 cdns3_rearm_transfer(priv_ep,
 959                                                      priv_ep->wa1_set);
 960                         }
 961                 }
 962         }
 963
 964         if ((ep_sts_reg & EP_STS_IOC) || (ep_sts_reg & EP_STS_ISP))
 965                 cdns3_transfer_completed(priv_dev, priv_ep);
 966
 967         return 0;
 968 }
 969
 970 static void cdns3_disconnect_gadget(struct cdns3_device *priv_dev)
 971 {
 972         if (priv_dev->gadget_driver && priv_dev->gadget_driver->disconnect) {
 973                 spin_unlock(&priv_dev->lock);
 974                 priv_dev->gadget_driver->disconnect(&priv_dev->gadget);
 975                 spin_lock(&priv_dev->lock);
 976         }
 977 }
 978
 979 /**
 980  * cdns3_check_usb_interrupt_proceed - Processes interrupt related to device
 981  * @priv_dev: extended gadget object
 982  * @usb_ists: bitmap representation of device's reported interrupts
 983  * (usb_ists register value)
 984  */
 985 static void cdns3_check_usb_interrupt_proceed(struct cdns3_device *priv_dev,
 986                                               u32 usb_ists)
 987 {
 988         int speed = 0;
 989
 990         trace_cdns3_usb_irq(priv_dev, usb_ists);
 991         if (usb_ists & USB_ISTS_L1ENTI) {
 992                 /*
 993                  * WORKAROUND: CDNS3 controller has issue with hardware resuming
 994                  * from L1. To fix it, if any DMA transfer is pending driver
 995                  * must starts driving resume signal immediately.
 996                  */
 997                 if (readl(&priv_dev->regs->drbl))
 998                         __cdns3_gadget_wakeup(priv_dev);
 999         }
1000
1001         /* Connection detected */
1002         if (usb_ists & (USB_ISTS_CON2I | USB_ISTS_CONI)) {
1003                 speed = cdns3_get_speed(priv_dev);
1004                 priv_dev->gadget.speed = speed;
1005                 usb_gadget_set_state(&priv_dev->gadget, USB_STATE_POWERED);
1006                 cdns3_ep0_config(priv_dev);
1007         }
1008
1009         /* Disconnection detected */
1010         if (usb_ists & (USB_ISTS_DIS2I | USB_ISTS_DISI)) {
1011                 cdns3_disconnect_gadget(priv_dev);
1012                 priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
1013                 usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);
1014                 cdns3_hw_reset_eps_config(priv_dev);
1015         }
1016
1017         if (usb_ists & (USB_ISTS_L2ENTI | USB_ISTS_U3ENTI)) {
1018                 if (priv_dev->gadget_driver &&
1019                     priv_dev->gadget_driver->suspend) {
1020                         spin_unlock(&priv_dev->lock);
1021                         priv_dev->gadget_driver->suspend(&priv_dev->gadget);
1022                         spin_lock(&priv_dev->lock);
1023                 }
1024         }
1025
1026         if (usb_ists & (USB_ISTS_L2EXTI | USB_ISTS_U3EXTI)) {
1027                 if (priv_dev->gadget_driver &&
1028                     priv_dev->gadget_driver->resume) {
1029                         spin_unlock(&priv_dev->lock);
1030                         priv_dev->gadget_driver->resume(&priv_dev->gadget);
1031                         spin_lock(&priv_dev->lock);
1032                 }
1033         }
1034
1035         /* reset*/
1036         if (usb_ists & (USB_ISTS_UWRESI | USB_ISTS_UHRESI | USB_ISTS_U2RESI)) {
1037                 if (priv_dev->gadget_driver) {
1038                         spin_unlock(&priv_dev->lock);
1039                         usb_gadget_udc_reset(&priv_dev->gadget,
1040                                              priv_dev->gadget_driver);
1041                         spin_lock(&priv_dev->lock);
1042
1043                         /*read again to check the actual speed*/
1044                         speed = cdns3_get_speed(priv_dev);
1045                         priv_dev->gadget.speed = speed;
1046                         cdns3_hw_reset_eps_config(priv_dev);
1047                         cdns3_ep0_config(priv_dev);
1048                 }
1049         }
1050 }
1051
1052 /**
1053  * cdns3_device_irq_handler- interrupt handler for device part of controller
1054  *
1055  * @irq: irq number for cdns3 core device
1056  * @data: structure of cdns3
1057  *
1058  * Returns IRQ_HANDLED or IRQ_NONE
1059  */
1060 static irqreturn_t cdns3_device_irq_handler(int irq, void *data)
1061 {
1062         struct cdns3_device *priv_dev;
1063         struct cdns3 *cdns = data;
1064         irqreturn_t ret = IRQ_NONE;
1065         u32 reg;
1066
1067         priv_dev = cdns->gadget_dev;
1068
1069         /* check USB device interrupt */
1070         reg = readl(&priv_dev->regs->usb_ists);
1071         if (reg) {
1072                 /* After masking interrupts the new interrupts won't be
1073                  * reported in usb_ists/ep_ists. In order to not lose some
1074                  * of them driver disables only detected interrupts.
1075                  * They will be enabled ASAP after clearing source of
1076                  * interrupt. This an unusual behavior only applies to
1077                  * usb_ists register.
1078                  */
1079                 reg = ~reg & readl(&priv_dev->regs->usb_ien);
1080                 /* mask deferred interrupt. */
1081                 writel(reg, &priv_dev->regs->usb_ien);
1082                 ret = IRQ_WAKE_THREAD;
1083         }
1084
1085         /* check endpoint interrupt */
1086         reg = readl(&priv_dev->regs->ep_ists);
1087         if (reg) {
1088                 writel(0, &priv_dev->regs->ep_ien);
1089                 ret = IRQ_WAKE_THREAD;
1090         }
1091
1092         return ret;
1093 }
1094
1095 /**
1096  * cdns3_device_thread_irq_handler- interrupt handler for device part
1097  * of controller
1098  *
1099  * @irq: irq number for cdns3 core device
1100  * @data: structure of cdns3
1101  *
1102  * Returns IRQ_HANDLED or IRQ_NONE
1103  */
1104 static irqreturn_t cdns3_device_thread_irq_handler(int irq, void *data)
1105 {
1106         struct cdns3_device *priv_dev;
1107         struct cdns3 *cdns = data;
1108         irqreturn_t ret = IRQ_NONE;
1109         unsigned long flags;
1110         int bit;
1111         u32 reg;
1112
1113         priv_dev = cdns->gadget_dev;
1114         spin_lock_irqsave(&priv_dev->lock, flags);
1115
1116         reg = readl(&priv_dev->regs->usb_ists);
1117         if (reg) {
1118                 writel(reg, &priv_dev->regs->usb_ists);
1119                 writel(USB_IEN_INIT, &priv_dev->regs->usb_ien);
1120                 cdns3_check_usb_interrupt_proceed(priv_dev, reg);
1121                 ret = IRQ_HANDLED;
1122         }
1123
1124         reg = readl(&priv_dev->regs->ep_ists);
1125
1126         /* handle default endpoint OUT */
1127         if (reg & EP_ISTS_EP_OUT0) {
1128                 cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_OUT);
1129                 ret = IRQ_HANDLED;
1130         }
1131
1132         /* handle default endpoint IN */
1133         if (reg & EP_ISTS_EP_IN0) {
1134                 cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_IN);
1135                 ret = IRQ_HANDLED;
1136         }
1137
1138         /* check if interrupt from non default endpoint, if no exit */
1139         reg &= ~(EP_ISTS_EP_OUT0 | EP_ISTS_EP_IN0);
1140         if (!reg)
1141                 goto irqend;
1142
1143         for_each_set_bit(bit, (unsigned long *)&reg,
1144                          sizeof(u32) * BITS_PER_BYTE) {
1145                 cdns3_check_ep_interrupt_proceed(priv_dev->eps[bit]);
1146                 ret = IRQ_HANDLED;
1147         }
1148
1149 irqend:
1150         writel(~0, &priv_dev->regs->ep_ien);
1151         spin_unlock_irqrestore(&priv_dev->lock, flags);
1152
1153         return ret;
1154 }
1155
1156 /**
1157  * cdns3_ep_onchip_buffer_reserve - Try to reserve onchip buf for EP
1158  *
1159  * The real reservation will occur during write to EP_CFG register,
1160  * this function is used to check if the 'size' reservation is allowed.
1161  *
1162  * @priv_dev: extended gadget object
1163  * @size: the size (KB) for EP would like to allocate
1164  * @is_in: endpoint direction
1165  *
1166  * Return 0 if the required size can met or negative value on failure
1167  */
1168 static int cdns3_ep_onchip_buffer_reserve(struct cdns3_device *priv_dev,
1169                                           int size, int is_in)
1170 {
1171         int remained;
1172
1173         /* 2KB are reserved for EP0*/
1174         remained = priv_dev->onchip_buffers - priv_dev->onchip_used_size - 2;
1175
1176         if (is_in) {
1177                 if (remained < size)
1178                         return -EPERM;
1179
1180                 priv_dev->onchip_used_size += size;
1181         } else {
1182                 int required;
1183
1184                 /**
1185                  *  ALL OUT EPs are shared the same chunk onchip memory, so
1186                  * driver checks if it already has assigned enough buffers
1187                  */
1188                 if (priv_dev->out_mem_is_allocated >= size)
1189                         return 0;
1190
1191                 required = size - priv_dev->out_mem_is_allocated;
1192
1193                 if (required > remained)
1194                         return -EPERM;
1195
1196                 priv_dev->out_mem_is_allocated += required;
1197                 priv_dev->onchip_used_size += required;
1198         }
1199
1200         return 0;
1201 }
1202
1203 void cdns3_configure_dmult(struct cdns3_device *priv_dev,
1204                            struct cdns3_endpoint *priv_ep)
1205 {
1206         struct cdns3_usb_regs __iomem *regs = priv_dev->regs;
1207
1208         /* For dev_ver > DEV_VER_V2 DMULT is configured per endpoint */
1209         if (priv_dev->dev_ver <= DEV_VER_V2)
1210                 writel(USB_CONF_DMULT, &regs->usb_conf);
1211
1212         if (priv_dev->dev_ver == DEV_VER_V2)
1213                 writel(USB_CONF2_EN_TDL_TRB, &regs->usb_conf2);
1214
1215         if (priv_dev->dev_ver >= DEV_VER_V3 && priv_ep) {
1216                 u32 mask;
1217
1218                 if (priv_ep->dir)
1219                         mask = BIT(priv_ep->num + 16);
1220                 else
1221                         mask = BIT(priv_ep->num);
1222
1223                 if (priv_ep->type != USB_ENDPOINT_XFER_ISOC) {
1224                         cdns3_set_register_bit(&regs->tdl_from_trb, mask);
1225                         cdns3_set_register_bit(&regs->tdl_beh, mask);
1226                         cdns3_set_register_bit(&regs->tdl_beh2, mask);
1227                         cdns3_set_register_bit(&regs->dma_adv_td, mask);
1228                 }
1229
1230                 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir)
1231                         cdns3_set_register_bit(&regs->tdl_from_trb, mask);
1232
1233                 cdns3_set_register_bit(&regs->dtrans, mask);
1234         }
1235 }
1236
1237 /**
1238  * cdns3_ep_config Configure hardware endpoint
1239  * @priv_ep: extended endpoint object
1240  */
1241 void cdns3_ep_config(struct cdns3_endpoint *priv_ep)
1242 {
1243         bool is_iso_ep = (priv_ep->type == USB_ENDPOINT_XFER_ISOC);
1244         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1245         u32 bEndpointAddress = priv_ep->num | priv_ep->dir;
1246         u32 max_packet_size = 0;
1247         u8 maxburst = 0;
1248         u32 ep_cfg = 0;
1249         u8 buffering;
1250         u8 mult = 0;
1251         int ret;
1252
1253         buffering = CDNS3_EP_BUF_SIZE - 1;
1254
1255         cdns3_configure_dmult(priv_dev, priv_ep);
1256
1257         switch (priv_ep->type) {
1258         case USB_ENDPOINT_XFER_INT:
1259                 ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_INT);
1260
1261                 if ((priv_dev->dev_ver == DEV_VER_V2 && !priv_ep->dir) ||
1262                     priv_dev->dev_ver > DEV_VER_V2)
1263                         ep_cfg |= EP_CFG_TDL_CHK;
1264                 break;
1265         case USB_ENDPOINT_XFER_BULK:
1266                 ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_BULK);
1267
1268                 if ((priv_dev->dev_ver == DEV_VER_V2  && !priv_ep->dir) ||
1269                     priv_dev->dev_ver > DEV_VER_V2)
1270                         ep_cfg |= EP_CFG_TDL_CHK;
1271                 break;
1272         default:
1273                 ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_ISOC);
1274                 mult = CDNS3_EP_ISO_HS_MULT - 1;
1275                 buffering = mult + 1;
1276         }
1277
1278         switch (priv_dev->gadget.speed) {
1279         case USB_SPEED_FULL:
1280                 max_packet_size = is_iso_ep ? 1023 : 64;
1281                 break;
1282         case USB_SPEED_HIGH:
1283                 max_packet_size = is_iso_ep ? 1024 : 512;
1284                 break;
1285         case USB_SPEED_SUPER:
1286                 /* It's limitation that driver assumes in driver. */
1287                 mult = 0;
1288                 max_packet_size = 1024;
1289                 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
1290                         maxburst = CDNS3_EP_ISO_SS_BURST - 1;
1291                         buffering = (mult + 1) *
1292                                     (maxburst + 1);
1293
1294                         if (priv_ep->interval > 1)
1295                                 buffering++;
1296                 } else {
1297                         maxburst = CDNS3_EP_BUF_SIZE - 1;
1298                 }
1299                 break;
1300         default:
1301                 /* all other speed are not supported */
1302                 return;
1303         }
1304
1305         if (max_packet_size == 1024)
1306                 priv_ep->trb_burst_size = 128;
1307         else if (max_packet_size >= 512)
1308                 priv_ep->trb_burst_size = 64;
1309         else
1310                 priv_ep->trb_burst_size = 16;
1311
1312         ret = cdns3_ep_onchip_buffer_reserve(priv_dev, buffering + 1,
1313                                              !!priv_ep->dir);
1314         if (ret) {
1315                 dev_err(priv_dev->dev, "onchip mem is full, ep is invalid\n");
1316                 return;
1317         }
1318
1319         ep_cfg |= EP_CFG_MAXPKTSIZE(max_packet_size) |
1320                   EP_CFG_MULT(mult) |
1321                   EP_CFG_BUFFERING(buffering) |
1322                   EP_CFG_MAXBURST(maxburst);
1323
1324         cdns3_select_ep(priv_dev, bEndpointAddress);
1325         writel(ep_cfg, &priv_dev->regs->ep_cfg);
1326
1327         dev_dbg(priv_dev->dev, "Configure %s: with val %08x\n",
1328                 priv_ep->name, ep_cfg);
1329 }
1330
1331 /* Find correct direction for HW endpoint according to description */
1332 static int cdns3_ep_dir_is_correct(struct usb_endpoint_descriptor *desc,
1333                                    struct cdns3_endpoint *priv_ep)
1334 {
1335         return (priv_ep->endpoint.caps.dir_in && usb_endpoint_dir_in(desc)) ||
1336                (priv_ep->endpoint.caps.dir_out && usb_endpoint_dir_out(desc));
1337 }
1338
1339 static struct
1340 cdns3_endpoint *cdns3_find_available_ep(struct cdns3_device *priv_dev,
1341                                         struct usb_endpoint_descriptor *desc)
1342 {
1343         struct usb_ep *ep;
1344         struct cdns3_endpoint *priv_ep;
1345
1346         list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
1347                 unsigned long num;
1348                 int ret;
1349                 /* ep name pattern likes epXin or epXout */
1350                 char c[2] = {ep->name[2], '\0'};
1351
1352                 ret = kstrtoul(c, 10, &num);
1353                 if (ret)
1354                         return ERR_PTR(ret);
1355
1356                 priv_ep = ep_to_cdns3_ep(ep);
1357                 if (cdns3_ep_dir_is_correct(desc, priv_ep)) {
1358                         if (!(priv_ep->flags & EP_CLAIMED)) {
1359                                 priv_ep->num  = num;
1360                                 return priv_ep;
1361                         }
1362                 }
1363         }
1364
1365         return ERR_PTR(-ENOENT);
1366 }
1367
1368 /*
1369  *  Cadence IP has one limitation that all endpoints must be configured
1370  * (Type & MaxPacketSize) before setting configuration through hardware
1371  * register, it means we can't change endpoints configuration after
1372  * set_configuration.
1373  *
1374  * This function set EP_CLAIMED flag which is added when the gadget driver
1375  * uses usb_ep_autoconfig to configure specific endpoint;
1376  * When the udc driver receives set_configurion request,
1377  * it goes through all claimed endpoints, and configure all endpoints
1378  * accordingly.
1379  *
1380  * At usb_ep_ops.enable/disable, we only enable and disable endpoint through
1381  * ep_cfg register which can be changed after set_configuration, and do
1382  * some software operation accordingly.
1383  */
1384 static struct
1385 usb_ep *cdns3_gadget_match_ep(struct usb_gadget *gadget,
1386                               struct usb_endpoint_descriptor *desc,
1387                               struct usb_ss_ep_comp_descriptor *comp_desc)
1388 {
1389         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
1390         struct cdns3_endpoint *priv_ep;
1391         unsigned long flags;
1392
1393         priv_ep = cdns3_find_available_ep(priv_dev, desc);
1394         if (IS_ERR(priv_ep)) {
1395                 dev_err(priv_dev->dev, "no available ep\n");
1396                 return NULL;
1397         }
1398
1399         dev_dbg(priv_dev->dev, "match endpoint: %s\n", priv_ep->name);
1400
1401         spin_lock_irqsave(&priv_dev->lock, flags);
1402         priv_ep->endpoint.desc = desc;
1403         priv_ep->dir  = usb_endpoint_dir_in(desc) ? USB_DIR_IN : USB_DIR_OUT;
1404         priv_ep->type = usb_endpoint_type(desc);
1405         priv_ep->flags |= EP_CLAIMED;
1406         priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
1407
1408         spin_unlock_irqrestore(&priv_dev->lock, flags);
1409         return &priv_ep->endpoint;
1410 }
1411
1412 /**
1413  * cdns3_gadget_ep_alloc_request Allocates request
1414  * @ep: endpoint object associated with request
1415  * @gfp_flags: gfp flags
1416  *
1417  * Returns allocated request address, NULL on allocation error
1418  */
1419 struct usb_request *cdns3_gadget_ep_alloc_request(struct usb_ep *ep,
1420                                                   gfp_t gfp_flags)
1421 {
1422         struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
1423         struct cdns3_request *priv_req;
1424
1425         priv_req = kzalloc(sizeof(*priv_req), gfp_flags);
1426         if (!priv_req)
1427                 return NULL;
1428
1429         priv_req->priv_ep = priv_ep;
1430
1431         trace_cdns3_alloc_request(priv_req);
1432         return &priv_req->request;
1433 }
1434
1435 /**
1436  * cdns3_gadget_ep_free_request Free memory occupied by request
1437  * @ep: endpoint object associated with request
1438  * @request: request to free memory
1439  */
1440 void cdns3_gadget_ep_free_request(struct usb_ep *ep,
1441                                   struct usb_request *request)
1442 {
1443         struct cdns3_request *priv_req = to_cdns3_request(request);
1444
1445         if (priv_req->aligned_buf)
1446                 priv_req->aligned_buf->in_use = 0;
1447
1448         trace_cdns3_free_request(priv_req);
1449         kfree(priv_req);
1450 }
1451
1452 /**
1453  * cdns3_gadget_ep_enable Enable endpoint
1454  * @ep: endpoint object
1455  * @desc: endpoint descriptor
1456  *
1457  * Returns 0 on success, error code elsewhere
1458  */
1459 static int cdns3_gadget_ep_enable(struct usb_ep *ep,
1460                                   const struct usb_endpoint_descriptor *desc)
1461 {
1462         struct cdns3_endpoint *priv_ep;
1463         struct cdns3_device *priv_dev;
1464         u32 reg = EP_STS_EN_TRBERREN;
1465         u32 bEndpointAddress;
1466         unsigned long flags;
1467         int enable = 1;
1468         int ret;
1469         int val;
1470
1471         priv_ep = ep_to_cdns3_ep(ep);
1472         priv_dev = priv_ep->cdns3_dev;
1473
1474         if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
1475                 dev_dbg(priv_dev->dev, "usbss: invalid parameters\n");
1476                 return -EINVAL;
1477         }
1478
1479         if (!desc->wMaxPacketSize) {
1480                 dev_err(priv_dev->dev, "usbss: missing wMaxPacketSize\n");
1481                 return -EINVAL;
1482         }
1483
1484         if (dev_WARN_ONCE(priv_dev->dev, priv_ep->flags & EP_ENABLED,
1485                           "%s is already enabled\n", priv_ep->name))
1486                 return 0;
1487
1488         spin_lock_irqsave(&priv_dev->lock, flags);
1489
1490         priv_ep->endpoint.desc = desc;
1491         priv_ep->type = usb_endpoint_type(desc);
1492         priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
1493
1494         if (priv_ep->interval > ISO_MAX_INTERVAL &&
1495             priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
1496                 dev_err(priv_dev->dev, "Driver is limited to %d period\n",
1497                         ISO_MAX_INTERVAL);
1498
1499                 ret =  -EINVAL;
1500                 goto exit;
1501         }
1502
1503         ret = cdns3_allocate_trb_pool(priv_ep);
1504
1505         if (ret)
1506                 goto exit;
1507
1508         bEndpointAddress = priv_ep->num | priv_ep->dir;
1509         cdns3_select_ep(priv_dev, bEndpointAddress);
1510
1511         trace_cdns3_gadget_ep_enable(priv_ep);
1512
1513         writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
1514
1515         ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
1516                                         !(val & (EP_CMD_CSTALL | EP_CMD_EPRST)),
1517                                         1, 1000);
1518
1519         if (unlikely(ret)) {
1520                 cdns3_free_trb_pool(priv_ep);
1521                 ret =  -EINVAL;
1522                 goto exit;
1523         }
1524
1525         /* enable interrupt for selected endpoint */
1526         cdns3_set_register_bit(&priv_dev->regs->ep_ien,
1527                                BIT(cdns3_ep_addr_to_index(bEndpointAddress)));
1528
1529         writel(reg, &priv_dev->regs->ep_sts_en);
1530
1531         /*
1532          * For some versions of controller at some point during ISO OUT traffic
1533          * DMA reads Transfer Ring for the EP which has never got doorbell.
1534          * This issue was detected only on simulation, but to avoid this issue
1535          * driver add protection against it. To fix it driver enable ISO OUT
1536          * endpoint before setting DRBL. This special treatment of ISO OUT
1537          * endpoints are recommended by controller specification.
1538          */
1539         if (priv_ep->type == USB_ENDPOINT_XFER_ISOC  && !priv_ep->dir)
1540                 enable = 0;
1541
1542         if (enable)
1543                 cdns3_set_register_bit(&priv_dev->regs->ep_cfg, EP_CFG_ENABLE);
1544
1545         ep->desc = desc;
1546         priv_ep->flags &= ~(EP_PENDING_REQUEST | EP_STALLED | EP_STALL_PENDING |
1547                             EP_QUIRK_ISO_OUT_EN);
1548         priv_ep->flags |= EP_ENABLED | EP_UPDATE_EP_TRBADDR;
1549         priv_ep->wa1_set = 0;
1550         priv_ep->enqueue = 0;
1551         priv_ep->dequeue = 0;
1552         reg = readl(&priv_dev->regs->ep_sts);
1553         priv_ep->pcs = !!EP_STS_CCS(reg);
1554         priv_ep->ccs = !!EP_STS_CCS(reg);
1555         /* one TRB is reserved for link TRB used in DMULT mode*/
1556         priv_ep->free_trbs = priv_ep->num_trbs - 1;
1557 exit:
1558         spin_unlock_irqrestore(&priv_dev->lock, flags);
1559
1560         return ret;
1561 }
1562
1563 /**
1564  * cdns3_gadget_ep_disable Disable endpoint
1565  * @ep: endpoint object
1566  *
1567  * Returns 0 on success, error code elsewhere
1568  */
1569 static int cdns3_gadget_ep_disable(struct usb_ep *ep)
1570 {
1571         struct cdns3_endpoint *priv_ep;
1572         struct cdns3_device *priv_dev;
1573         struct usb_request *request;
1574         unsigned long flags;
1575         int ret = 0;
1576         u32 ep_cfg;
1577         int val;
1578
1579         if (!ep) {
1580                 pr_err("usbss: invalid parameters\n");
1581                 return -EINVAL;
1582         }
1583
1584         priv_ep = ep_to_cdns3_ep(ep);
1585         priv_dev = priv_ep->cdns3_dev;
1586
1587         if (dev_WARN_ONCE(priv_dev->dev, !(priv_ep->flags & EP_ENABLED),
1588                           "%s is already disabled\n", priv_ep->name))
1589                 return 0;
1590
1591         spin_lock_irqsave(&priv_dev->lock, flags);
1592
1593         trace_cdns3_gadget_ep_disable(priv_ep);
1594
1595         cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
1596
1597         ep_cfg = readl(&priv_dev->regs->ep_cfg);
1598         ep_cfg &= ~EP_CFG_ENABLE;
1599         writel(ep_cfg, &priv_dev->regs->ep_cfg);
1600
1601         /**
1602          * Driver needs some time before resetting endpoint.
1603          * It need waits for clearing DBUSY bit or for timeout expired.
1604          * 10us is enough time for controller to stop transfer.
1605          */
1606         readl_poll_timeout_atomic(&priv_dev->regs->ep_sts, val,
1607                                   !(val & EP_STS_DBUSY), 1, 10);
1608         writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
1609
1610         readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
1611                                   !(val & (EP_CMD_CSTALL | EP_CMD_EPRST)),
1612                                   1, 1000);
1613         if (unlikely(ret))
1614                 dev_err(priv_dev->dev, "Timeout: %s resetting failed.\n",
1615                         priv_ep->name);
1616
1617         while (!list_empty(&priv_ep->pending_req_list)) {
1618                 request = cdns3_next_request(&priv_ep->pending_req_list);
1619
1620                 cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
1621                                       -ESHUTDOWN);
1622         }
1623
1624         while (!list_empty(&priv_ep->deferred_req_list)) {
1625                 request = cdns3_next_request(&priv_ep->deferred_req_list);
1626
1627                 cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
1628                                       -ESHUTDOWN);
1629         }
1630
1631         ep->desc = NULL;
1632         priv_ep->flags &= ~EP_ENABLED;
1633
1634         spin_unlock_irqrestore(&priv_dev->lock, flags);
1635
1636         return ret;
1637 }
1638
1639 /**
1640  * cdns3_gadget_ep_queue Transfer data on endpoint
1641  * @ep: endpoint object
1642  * @request: request object
1643  * @gfp_flags: gfp flags
1644  *
1645  * Returns 0 on success, error code elsewhere
1646  */
1647 static int __cdns3_gadget_ep_queue(struct usb_ep *ep,
1648                                    struct usb_request *request,
1649                                    gfp_t gfp_flags)
1650 {
1651         struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
1652         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1653         struct cdns3_request *priv_req;
1654         int ret = 0;
1655
1656         request->actual = 0;
1657         request->status = -EINPROGRESS;
1658         priv_req = to_cdns3_request(request);
1659         trace_cdns3_ep_queue(priv_req);
1660
1661         ret = cdns3_prepare_aligned_request_buf(priv_req);
1662         if (ret < 0)
1663                 return ret;
1664
1665         ret = usb_gadget_map_request_by_dev(priv_dev->sysdev, request,
1666                                             usb_endpoint_dir_in(ep->desc));
1667         if (ret)
1668                 return ret;
1669
1670         list_add_tail(&request->list, &priv_ep->deferred_req_list);
1671
1672         /*
1673          * If hardware endpoint configuration has not been set yet then
1674          * just queue request in deferred list. Transfer will be started in
1675          * cdns3_set_hw_configuration.
1676          */
1677         if (priv_dev->hw_configured_flag && !(priv_ep->flags & EP_STALLED) &&
1678             !(priv_ep->flags & EP_STALL_PENDING))
1679                 cdns3_start_all_request(priv_dev, priv_ep);
1680
1681         return 0;
1682 }
1683
1684 static int cdns3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
1685                                  gfp_t gfp_flags)
1686 {
1687         struct usb_request *zlp_request;
1688         struct cdns3_endpoint *priv_ep;
1689         struct cdns3_device *priv_dev;
1690         unsigned long flags;
1691         int ret;
1692
1693         if (!request || !ep)
1694                 return -EINVAL;
1695
1696         priv_ep = ep_to_cdns3_ep(ep);
1697         priv_dev = priv_ep->cdns3_dev;
1698
1699         spin_lock_irqsave(&priv_dev->lock, flags);
1700
1701         ret = __cdns3_gadget_ep_queue(ep, request, gfp_flags);
1702
1703         if (ret == 0 && request->zero && request->length &&
1704             (request->length % ep->maxpacket == 0)) {
1705                 struct cdns3_request *priv_req;
1706
1707                 zlp_request = cdns3_gadget_ep_alloc_request(ep, GFP_ATOMIC);
1708                 zlp_request->buf = priv_dev->zlp_buf;
1709                 zlp_request->length = 0;
1710
1711                 priv_req = to_cdns3_request(zlp_request);
1712                 priv_req->flags |= REQUEST_ZLP;
1713
1714                 dev_dbg(priv_dev->dev, "Queuing ZLP for endpoint: %s\n",
1715                         priv_ep->name);
1716                 ret = __cdns3_gadget_ep_queue(ep, zlp_request, gfp_flags);
1717         }
1718
1719         spin_unlock_irqrestore(&priv_dev->lock, flags);
1720         return ret;
1721 }
1722
1723 /**
1724  * cdns3_gadget_ep_dequeue Remove request from transfer queue
1725  * @ep: endpoint object associated with request
1726  * @request: request object
1727  *
1728  * Returns 0 on success, error code elsewhere
1729  */
1730 int cdns3_gadget_ep_dequeue(struct usb_ep *ep,
1731                             struct usb_request *request)
1732 {
1733         struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
1734         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1735         struct usb_request *req, *req_temp;
1736         struct cdns3_request *priv_req;
1737         struct cdns3_trb *link_trb;
1738         unsigned long flags;
1739         int ret = 0;
1740
1741         if (!ep || !request || !ep->desc)
1742                 return -EINVAL;
1743
1744         spin_lock_irqsave(&priv_dev->lock, flags);
1745
1746         priv_req = to_cdns3_request(request);
1747
1748         trace_cdns3_ep_dequeue(priv_req);
1749
1750         cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
1751
1752         list_for_each_entry_safe(req, req_temp, &priv_ep->pending_req_list,
1753                                  list) {
1754                 if (request == req)
1755                         goto found;
1756         }
1757
1758         list_for_each_entry_safe(req, req_temp, &priv_ep->deferred_req_list,
1759                                  list) {
1760                 if (request == req)
1761                         goto found;
1762         }
1763
1764         goto not_found;
1765
1766 found:
1767
1768         if (priv_ep->wa1_trb == priv_req->trb)
1769                 cdns3_wa1_restore_cycle_bit(priv_ep);
1770
1771         link_trb = priv_req->trb;
1772         cdns3_move_deq_to_next_trb(priv_req);
1773         cdns3_gadget_giveback(priv_ep, priv_req, -ECONNRESET);
1774
1775         /* Update ring */
1776         request = cdns3_next_request(&priv_ep->deferred_req_list);
1777         if (request) {
1778                 priv_req = to_cdns3_request(request);
1779
1780                 link_trb->buffer = TRB_BUFFER(priv_ep->trb_pool_dma +
1781                                               (priv_req->start_trb * TRB_SIZE));
1782                 link_trb->control = (link_trb->control & TRB_CYCLE) |
1783                                     TRB_TYPE(TRB_LINK) | TRB_CHAIN | TRB_TOGGLE;
1784         } else {
1785                 priv_ep->flags |= EP_UPDATE_EP_TRBADDR;
1786         }
1787
1788 not_found:
1789         spin_unlock_irqrestore(&priv_dev->lock, flags);
1790         return ret;
1791 }
1792
1793 /**
1794  * __cdns3_gadget_ep_set_halt Sets stall on selected endpoint
1795  * Should be called after acquiring spin_lock and selecting ep
1796  * @ep: endpoint object to set stall on.
1797  */
1798 void __cdns3_gadget_ep_set_halt(struct cdns3_endpoint *priv_ep)
1799 {
1800         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1801
1802         trace_cdns3_halt(priv_ep, 1, 0);
1803
1804         if (!(priv_ep->flags & EP_STALLED)) {
1805                 u32 ep_sts_reg = readl(&priv_dev->regs->ep_sts);
1806
1807                 if (!(ep_sts_reg & EP_STS_DBUSY))
1808                         cdns3_ep_stall_flush(priv_ep);
1809                 else
1810                         priv_ep->flags |= EP_STALL_PENDING;
1811         }
1812 }
1813
1814 /**
1815  * __cdns3_gadget_ep_clear_halt Clears stall on selected endpoint
1816  * Should be called after acquiring spin_lock and selecting ep
1817  * @ep: endpoint object to clear stall on
1818  */
1819 int __cdns3_gadget_ep_clear_halt(struct cdns3_endpoint *priv_ep)
1820 {
1821         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1822         struct usb_request *request;
1823         int ret = 0;
1824         int val;
1825
1826         trace_cdns3_halt(priv_ep, 0, 0);
1827
1828         writel(EP_CMD_CSTALL | EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
1829
1830         /* wait for EPRST cleared */
1831         readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
1832                                   !(val & EP_CMD_EPRST), 1, 100);
1833         if (ret)
1834                 return -EINVAL;
1835
1836         priv_ep->flags &= ~(EP_STALLED | EP_STALL_PENDING);
1837
1838         request = cdns3_next_request(&priv_ep->pending_req_list);
1839
1840         if (request)
1841                 cdns3_rearm_transfer(priv_ep, 1);
1842
1843         cdns3_start_all_request(priv_dev, priv_ep);
1844         return ret;
1845 }
1846
1847 /**
1848  * cdns3_gadget_ep_set_halt Sets/clears stall on selected endpoint
1849  * @ep: endpoint object to set/clear stall on
1850  * @value: 1 for set stall, 0 for clear stall
1851  *
1852  * Returns 0 on success, error code elsewhere
1853  */
1854 int cdns3_gadget_ep_set_halt(struct usb_ep *ep, int value)
1855 {
1856         struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
1857         struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1858         unsigned long flags;
1859         int ret = 0;
1860
1861         if (!(priv_ep->flags & EP_ENABLED))
1862                 return -EPERM;
1863
1864         spin_lock_irqsave(&priv_dev->lock, flags);
1865
1866         cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
1867
1868         if (!value) {
1869                 priv_ep->flags &= ~EP_WEDGE;
1870                 ret = __cdns3_gadget_ep_clear_halt(priv_ep);
1871         } else {
1872                 __cdns3_gadget_ep_set_halt(priv_ep);
1873         }
1874
1875         spin_unlock_irqrestore(&priv_dev->lock, flags);
1876
1877         return ret;
1878 }
1879
1880 extern const struct usb_ep_ops cdns3_gadget_ep0_ops;
1881
1882 static const struct usb_ep_ops cdns3_gadget_ep_ops = {
1883         .enable = cdns3_gadget_ep_enable,
1884         .disable = cdns3_gadget_ep_disable,
1885         .alloc_request = cdns3_gadget_ep_alloc_request,
1886         .free_request = cdns3_gadget_ep_free_request,
1887         .queue = cdns3_gadget_ep_queue,
1888         .dequeue = cdns3_gadget_ep_dequeue,
1889         .set_halt = cdns3_gadget_ep_set_halt,
1890         .set_wedge = cdns3_gadget_ep_set_wedge,
1891 };
1892
1893 /**
1894  * cdns3_gadget_get_frame Returns number of actual ITP frame
1895  * @gadget: gadget object
1896  *
1897  * Returns number of actual ITP frame
1898  */
1899 static int cdns3_gadget_get_frame(struct usb_gadget *gadget)
1900 {
1901         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
1902
1903         return readl(&priv_dev->regs->usb_itpn);
1904 }
1905
1906 int __cdns3_gadget_wakeup(struct cdns3_device *priv_dev)
1907 {
1908         enum usb_device_speed speed;
1909
1910         speed = cdns3_get_speed(priv_dev);
1911
1912         if (speed >= USB_SPEED_SUPER)
1913                 return 0;
1914
1915         /* Start driving resume signaling to indicate remote wakeup. */
1916         writel(USB_CONF_LGO_L0, &priv_dev->regs->usb_conf);
1917
1918         return 0;
1919 }
1920
1921 static int cdns3_gadget_wakeup(struct usb_gadget *gadget)
1922 {
1923         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
1924         unsigned long flags;
1925         int ret = 0;
1926
1927         spin_lock_irqsave(&priv_dev->lock, flags);
1928         ret = __cdns3_gadget_wakeup(priv_dev);
1929         spin_unlock_irqrestore(&priv_dev->lock, flags);
1930         return ret;
1931 }
1932
1933 static int cdns3_gadget_set_selfpowered(struct usb_gadget *gadget,
1934                                         int is_selfpowered)
1935 {
1936         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
1937         unsigned long flags;
1938
1939         spin_lock_irqsave(&priv_dev->lock, flags);
1940         priv_dev->is_selfpowered = !!is_selfpowered;
1941         spin_unlock_irqrestore(&priv_dev->lock, flags);
1942         return 0;
1943 }
1944
1945 static int cdns3_gadget_pullup(struct usb_gadget *gadget, int is_on)
1946 {
1947         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
1948
1949         if (is_on)
1950                 writel(USB_CONF_DEVEN, &priv_dev->regs->usb_conf);
1951         else
1952                 writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);
1953
1954         return 0;
1955 }
1956
1957 static void cdns3_gadget_config(struct cdns3_device *priv_dev)
1958 {
1959         struct cdns3_usb_regs __iomem *regs = priv_dev->regs;
1960         u32 reg;
1961
1962         cdns3_ep0_config(priv_dev);
1963
1964         /* enable interrupts for endpoint 0 (in and out) */
1965         writel(EP_IEN_EP_OUT0 | EP_IEN_EP_IN0, &regs->ep_ien);
1966
1967         /*
1968          * Driver needs to modify LFPS minimal U1 Exit time for DEV_VER_TI_V1
1969          * revision of controller.
1970          */
1971         if (priv_dev->dev_ver == DEV_VER_TI_V1) {
1972                 reg = readl(&regs->dbg_link1);
1973
1974                 reg &= ~DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_MASK;
1975                 reg |= DBG_LINK1_LFPS_MIN_GEN_U1_EXIT(0x55) |
1976                        DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_SET;
1977                 writel(reg, &regs->dbg_link1);
1978         }
1979
1980         /*
1981          * By default some platforms has set protected access to memory.
1982          * This cause problem with cache, so driver restore non-secure
1983          * access to memory.
1984          */
1985         reg = readl(&regs->dma_axi_ctrl);
1986         reg |= DMA_AXI_CTRL_MARPROT(DMA_AXI_CTRL_NON_SECURE) |
1987                DMA_AXI_CTRL_MAWPROT(DMA_AXI_CTRL_NON_SECURE);
1988         writel(reg, &regs->dma_axi_ctrl);
1989
1990         /* enable generic interrupt*/
1991         writel(USB_IEN_INIT, &regs->usb_ien);
1992         writel(USB_CONF_CLK2OFFDS | USB_CONF_L1DS, &regs->usb_conf);
1993
1994         cdns3_configure_dmult(priv_dev, NULL);
1995
1996         cdns3_gadget_pullup(&priv_dev->gadget, 1);
1997 }
1998
1999 /**
2000  * cdns3_gadget_udc_start Gadget start
2001  * @gadget: gadget object
2002  * @driver: driver which operates on this gadget
2003  *
2004  * Returns 0 on success, error code elsewhere
2005  */
2006 static int cdns3_gadget_udc_start(struct usb_gadget *gadget,
2007                                   struct usb_gadget_driver *driver)
2008 {
2009         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2010         unsigned long flags;
2011
2012         spin_lock_irqsave(&priv_dev->lock, flags);
2013         priv_dev->gadget_driver = driver;
2014         cdns3_gadget_config(priv_dev);
2015         spin_unlock_irqrestore(&priv_dev->lock, flags);
2016         return 0;
2017 }
2018
2019 /**
2020  * cdns3_gadget_udc_stop Stops gadget
2021  * @gadget: gadget object
2022  *
2023  * Returns 0
2024  */
2025 static int cdns3_gadget_udc_stop(struct usb_gadget *gadget)
2026 {
2027         struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2028         struct cdns3_endpoint *priv_ep;
2029         u32 bEndpointAddress;
2030         struct usb_ep *ep;
2031         int ret = 0;
2032         int val;
2033
2034         priv_dev->gadget_driver = NULL;
2035
2036         priv_dev->onchip_used_size = 0;
2037         priv_dev->out_mem_is_allocated = 0;
2038         priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
2039
2040         list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
2041                 priv_ep = ep_to_cdns3_ep(ep);
2042                 bEndpointAddress = priv_ep->num | priv_ep->dir;
2043                 cdns3_select_ep(priv_dev, bEndpointAddress);
2044                 writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2045                 readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2046                                           !(val & EP_CMD_EPRST), 1, 100);
2047         }
2048
2049         /* disable interrupt for device */
2050         writel(0, &priv_dev->regs->usb_ien);
2051         writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);
2052
2053         return ret;
2054 }
2055
2056 static const struct usb_gadget_ops cdns3_gadget_ops = {
2057         .get_frame = cdns3_gadget_get_frame,
2058         .wakeup = cdns3_gadget_wakeup,
2059         .set_selfpowered = cdns3_gadget_set_selfpowered,
2060         .pullup = cdns3_gadget_pullup,
2061         .udc_start = cdns3_gadget_udc_start,
2062         .udc_stop = cdns3_gadget_udc_stop,
2063         .match_ep = cdns3_gadget_match_ep,
2064 };
2065
2066 static void cdns3_free_all_eps(struct cdns3_device *priv_dev)
2067 {
2068         int i;
2069
2070         /* ep0 OUT point to ep0 IN. */
2071         priv_dev->eps[16] = NULL;
2072
2073         for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++)
2074                 if (priv_dev->eps[i]) {
2075                         cdns3_free_trb_pool(priv_dev->eps[i]);
2076                         devm_kfree(priv_dev->dev, priv_dev->eps[i]);
2077                 }
2078 }
2079
2080 /**
2081  * cdns3_init_eps Initializes software endpoints of gadget
2082  * @cdns3: extended gadget object
2083  *
2084  * Returns 0 on success, error code elsewhere
2085  */
2086 static int cdns3_init_eps(struct cdns3_device *priv_dev)
2087 {
2088         u32 ep_enabled_reg, iso_ep_reg;
2089         struct cdns3_endpoint *priv_ep;
2090         int ep_dir, ep_number;
2091         u32 ep_mask;
2092         int ret = 0;
2093         int i;
2094
2095         /* Read it from USB_CAP3 to USB_CAP5 */
2096         ep_enabled_reg = readl(&priv_dev->regs->usb_cap3);
2097         iso_ep_reg = readl(&priv_dev->regs->usb_cap4);
2098
2099         dev_dbg(priv_dev->dev, "Initializing non-zero endpoints\n");
2100
2101         for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++) {
2102                 ep_dir = i >> 4;        /* i div 16 */
2103                 ep_number = i & 0xF;    /* i % 16 */
2104                 ep_mask = BIT(i);
2105
2106                 if (!(ep_enabled_reg & ep_mask))
2107                         continue;
2108
2109                 if (ep_dir && !ep_number) {
2110                         priv_dev->eps[i] = priv_dev->eps[0];
2111                         continue;
2112                 }
2113
2114                 priv_ep = devm_kzalloc(priv_dev->dev, sizeof(*priv_ep),
2115                                        GFP_KERNEL);
2116                 if (!priv_ep) {
2117                         ret = -ENOMEM;
2118                         goto err;
2119                 }
2120
2121                 /* set parent of endpoint object */
2122                 priv_ep->cdns3_dev = priv_dev;
2123                 priv_dev->eps[i] = priv_ep;
2124                 priv_ep->num = ep_number;
2125                 priv_ep->dir = ep_dir ? USB_DIR_IN : USB_DIR_OUT;
2126
2127                 if (!ep_number) {
2128                         ret = cdns3_init_ep0(priv_dev, priv_ep);
2129                         if (ret) {
2130                                 dev_err(priv_dev->dev, "Failed to init ep0\n");
2131                                 goto err;
2132                         }
2133                 } else {
2134                         snprintf(priv_ep->name, sizeof(priv_ep->name), "ep%d%s",
2135                                  ep_number, !!ep_dir ? "in" : "out");
2136                         priv_ep->endpoint.name = priv_ep->name;
2137
2138                         usb_ep_set_maxpacket_limit(&priv_ep->endpoint,
2139                                                    CDNS3_EP_MAX_PACKET_LIMIT);
2140                         priv_ep->endpoint.max_streams = CDNS3_EP_MAX_STREAMS;
2141                         priv_ep->endpoint.ops = &cdns3_gadget_ep_ops;
2142                         if (ep_dir)
2143                                 priv_ep->endpoint.caps.dir_in = 1;
2144                         else
2145                                 priv_ep->endpoint.caps.dir_out = 1;
2146
2147                         if (iso_ep_reg & ep_mask)
2148                                 priv_ep->endpoint.caps.type_iso = 1;
2149
2150                         priv_ep->endpoint.caps.type_bulk = 1;
2151                         priv_ep->endpoint.caps.type_int = 1;
2152
2153                         list_add_tail(&priv_ep->endpoint.ep_list,
2154                                       &priv_dev->gadget.ep_list);
2155                 }
2156
2157                 priv_ep->flags = 0;
2158
2159                 dev_info(priv_dev->dev, "Initialized  %s support: %s %s\n",
2160                          priv_ep->name,
2161                          priv_ep->endpoint.caps.type_bulk ? "BULK, INT" : "",
2162                          priv_ep->endpoint.caps.type_iso ? "ISO" : "");
2163
2164                 INIT_LIST_HEAD(&priv_ep->pending_req_list);
2165                 INIT_LIST_HEAD(&priv_ep->deferred_req_list);
2166         }
2167
2168         return 0;
2169 err:
2170         cdns3_free_all_eps(priv_dev);
2171         return -ENOMEM;
2172 }
2173
2174 void cdns3_gadget_exit(struct cdns3 *cdns)
2175 {
2176         struct cdns3_device *priv_dev;
2177
2178         priv_dev = cdns->gadget_dev;
2179
2180         devm_free_irq(cdns->dev, cdns->dev_irq, cdns);
2181
2182         pm_runtime_mark_last_busy(cdns->dev);
2183         pm_runtime_put_autosuspend(cdns->dev);
2184
2185         usb_del_gadget_udc(&priv_dev->gadget);
2186
2187         cdns3_free_all_eps(priv_dev);
2188
2189         while (!list_empty(&priv_dev->aligned_buf_list)) {
2190                 struct cdns3_aligned_buf *buf;
2191
2192                 buf = cdns3_next_align_buf(&priv_dev->aligned_buf_list);
2193                 dma_free_coherent(priv_dev->sysdev, buf->size,
2194                                   buf->buf,
2195                                   buf->dma);
2196
2197                 list_del(&buf->list);
2198                 kfree(buf);
2199         }
2200
2201         dma_free_coherent(priv_dev->sysdev, 8, priv_dev->setup_buf,
2202                           priv_dev->setup_dma);
2203
2204         kfree(priv_dev->zlp_buf);
2205         kfree(priv_dev);
2206         cdns->gadget_dev = NULL;
2207         cdns3_drd_switch_gadget(cdns, 0);
2208 }
2209
2210 static int cdns3_gadget_start(struct cdns3 *cdns)
2211 {
2212         struct cdns3_device *priv_dev;
2213         u32 max_speed;
2214         int ret;
2215
2216         priv_dev = kzalloc(sizeof(*priv_dev), GFP_KERNEL);
2217         if (!priv_dev)
2218                 return -ENOMEM;
2219
2220         cdns->gadget_dev = priv_dev;
2221         priv_dev->sysdev = cdns->dev;
2222         priv_dev->dev = cdns->dev;
2223         priv_dev->regs = cdns->dev_regs;
2224
2225         device_property_read_u16(priv_dev->dev, "cdns,on-chip-buff-size",
2226                                  &priv_dev->onchip_buffers);
2227
2228         if (priv_dev->onchip_buffers <=  0) {
2229                 u32 reg = readl(&priv_dev->regs->usb_cap2);
2230
2231                 priv_dev->onchip_buffers = USB_CAP2_ACTUAL_MEM_SIZE(reg);
2232         }
2233
2234         if (!priv_dev->onchip_buffers)
2235                 priv_dev->onchip_buffers = 256;
2236
2237         max_speed = usb_get_maximum_speed(cdns->dev);
2238
2239         /* Check the maximum_speed parameter */
2240         switch (max_speed) {
2241         case USB_SPEED_FULL:
2242                 writel(USB_CONF_SFORCE_FS, &priv_dev->regs->usb_conf);
2243                 break;
2244         case USB_SPEED_HIGH:
2245                 writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf);
2246                 break;
2247         case USB_SPEED_SUPER:
2248                 break;
2249         default:
2250                 dev_err(cdns->dev, "invalid maximum_speed parameter %d\n",
2251                         max_speed);
2252                 /* fall through */
2253         case USB_SPEED_UNKNOWN:
2254                 /* default to superspeed */
2255                 max_speed = USB_SPEED_SUPER;
2256                 break;
2257         }
2258
2259         /* fill gadget fields */
2260         priv_dev->gadget.max_speed = max_speed;
2261         priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
2262         priv_dev->gadget.ops = &cdns3_gadget_ops;
2263         priv_dev->gadget.name = "usb-ss-gadget";
2264         priv_dev->gadget.sg_supported = 1;
2265         priv_dev->gadget.quirk_avoids_skb_reserve = 1;
2266
2267         spin_lock_init(&priv_dev->lock);
2268         INIT_WORK(&priv_dev->pending_status_wq,
2269                   cdns3_pending_setup_status_handler);
2270
2271         INIT_WORK(&priv_dev->aligned_buf_wq,
2272                   cdns3_free_aligned_request_buf);
2273
2274         /* initialize endpoint container */
2275         INIT_LIST_HEAD(&priv_dev->gadget.ep_list);
2276         INIT_LIST_HEAD(&priv_dev->aligned_buf_list);
2277
2278         ret = cdns3_init_eps(priv_dev);
2279         if (ret) {
2280                 dev_err(priv_dev->dev, "Failed to create endpoints\n");
2281                 goto err1;
2282         }
2283
2284         /* allocate memory for setup packet buffer */
2285         priv_dev->setup_buf = dma_alloc_coherent(priv_dev->sysdev, 8,
2286                                                  &priv_dev->setup_dma, GFP_DMA);
2287         if (!priv_dev->setup_buf) {
2288                 ret = -ENOMEM;
2289                 goto err2;
2290         }
2291
2292         priv_dev->dev_ver = readl(&priv_dev->regs->usb_cap6);
2293
2294         dev_dbg(priv_dev->dev, "Device Controller version: %08x\n",
2295                 readl(&priv_dev->regs->usb_cap6));
2296         dev_dbg(priv_dev->dev, "USB Capabilities:: %08x\n",
2297                 readl(&priv_dev->regs->usb_cap1));
2298         dev_dbg(priv_dev->dev, "On-Chip memory cnfiguration: %08x\n",
2299                 readl(&priv_dev->regs->usb_cap2));
2300
2301         priv_dev->dev_ver = GET_DEV_BASE_VERSION(priv_dev->dev_ver);
2302
2303         priv_dev->zlp_buf = kzalloc(CDNS3_EP_ZLP_BUF_SIZE, GFP_KERNEL);
2304         if (!priv_dev->zlp_buf) {
2305                 ret = -ENOMEM;
2306                 goto err3;
2307         }
2308
2309         /* add USB gadget device */
2310         ret = usb_add_gadget_udc(priv_dev->dev, &priv_dev->gadget);
2311         if (ret < 0) {
2312                 dev_err(priv_dev->dev,
2313                         "Failed to register USB device controller\n");
2314                 goto err4;
2315         }
2316
2317         return 0;
2318 err4:
2319         kfree(priv_dev->zlp_buf);
2320 err3:
2321         dma_free_coherent(priv_dev->sysdev, 8, priv_dev->setup_buf,
2322                           priv_dev->setup_dma);
2323 err2:
2324         cdns3_free_all_eps(priv_dev);
2325 err1:
2326         cdns->gadget_dev = NULL;
2327         return ret;
2328 }
2329
2330 static int __cdns3_gadget_init(struct cdns3 *cdns)
2331 {
2332         struct cdns3_device *priv_dev;
2333         int ret = 0;
2334
2335         cdns3_drd_switch_gadget(cdns, 1);
2336         pm_runtime_get_sync(cdns->dev);
2337
2338         ret = cdns3_gadget_start(cdns);
2339         if (ret)
2340                 return ret;
2341
2342         priv_dev = cdns->gadget_dev;
2343
2344         /*
2345          * Because interrupt line can be shared with other components in
2346          * driver it can't use IRQF_ONESHOT flag here.
2347          */
2348         ret = devm_request_threaded_irq(cdns->dev, cdns->dev_irq,
2349                                         cdns3_device_irq_handler,
2350                                         cdns3_device_thread_irq_handler,
2351                                         IRQF_SHARED, dev_name(cdns->dev), cdns);
2352
2353         if (ret)
2354                 goto err0;
2355
2356         return 0;
2357 err0:
2358         cdns3_gadget_exit(cdns);
2359         return ret;
2360 }
2361
2362 static int cdns3_gadget_suspend(struct cdns3 *cdns, bool do_wakeup)
2363 {
2364         struct cdns3_device *priv_dev = cdns->gadget_dev;
2365
2366         cdns3_disconnect_gadget(priv_dev);
2367
2368         priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
2369         usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);
2370         cdns3_hw_reset_eps_config(priv_dev);
2371
2372         /* disable interrupt for device */
2373         writel(0, &priv_dev->regs->usb_ien);
2374
2375         cdns3_gadget_pullup(&priv_dev->gadget, 0);
2376
2377         return 0;
2378 }
2379
2380 static int cdns3_gadget_resume(struct cdns3 *cdns, bool hibernated)
2381 {
2382         struct cdns3_device *priv_dev = cdns->gadget_dev;
2383
2384         if (!priv_dev->gadget_driver)
2385                 return 0;
2386
2387         cdns3_gadget_config(priv_dev);
2388
2389         return 0;
2390 }
2391
2392 /**
2393  * cdns3_gadget_init - initialize device structure
2394  *
2395  * cdns: cdns3 instance
2396  *
2397  * This function initializes the gadget.
2398  */
2399 int cdns3_gadget_init(struct cdns3 *cdns)
2400 {
2401         struct cdns3_role_driver *rdrv;
2402
2403         rdrv = devm_kzalloc(cdns->dev, sizeof(*rdrv), GFP_KERNEL);
2404         if (!rdrv)
2405                 return -ENOMEM;
2406
2407         rdrv->start     = __cdns3_gadget_init;
2408         rdrv->stop      = cdns3_gadget_exit;
2409         rdrv->suspend   = cdns3_gadget_suspend;
2410         rdrv->resume    = cdns3_gadget_resume;
2411         rdrv->state     = CDNS3_ROLE_STATE_INACTIVE;
2412         rdrv->name      = "gadget";
2413         cdns->roles[USB_ROLE_DEVICE] = rdrv;
2414
2415         return 0;
2416 }