drivers/usb/chipidea/udc.c

   1 /*
   2  * udc.c - ChipIdea UDC driver
   3  *
   4  * Copyright (C) 2008 Chipidea - MIPS Technologies, Inc. All rights reserved.
   5  *
   6  * Author: David Lopo
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License version 2 as
  10  * published by the Free Software Foundation.
  11  */
  12
  13 #include <linux/delay.h>
  14 #include <linux/device.h>
  15 #include <linux/dmapool.h>
  16 #include <linux/dma-mapping.h>
  17 #include <linux/err.h>
  18 #include <linux/init.h>
  19 #include <linux/platform_device.h>
  20 #include <linux/module.h>
  21 #include <linux/interrupt.h>
  22 #include <linux/io.h>
  23 #include <linux/irq.h>
  24 #include <linux/kernel.h>
  25 #include <linux/slab.h>
  26 #include <linux/pm_runtime.h>
  27 #include <linux/usb/ch9.h>
  28 #include <linux/usb/gadget.h>
  29 #include <linux/usb/otg.h>
  30 #include <linux/usb/chipidea.h>
  31
  32 #include "ci.h"
  33 #include "udc.h"
  34 #include "bits.h"
  35 #include "debug.h"
  36
  37 /* control endpoint description */
  38 static const struct usb_endpoint_descriptor
  39 ctrl_endpt_out_desc = {
  40         .bLength         = USB_DT_ENDPOINT_SIZE,
  41         .bDescriptorType = USB_DT_ENDPOINT,
  42
  43         .bEndpointAddress = USB_DIR_OUT,
  44         .bmAttributes    = USB_ENDPOINT_XFER_CONTROL,
  45         .wMaxPacketSize  = cpu_to_le16(CTRL_PAYLOAD_MAX),
  46 };
  47
  48 static const struct usb_endpoint_descriptor
  49 ctrl_endpt_in_desc = {
  50         .bLength         = USB_DT_ENDPOINT_SIZE,
  51         .bDescriptorType = USB_DT_ENDPOINT,
  52
  53         .bEndpointAddress = USB_DIR_IN,
  54         .bmAttributes    = USB_ENDPOINT_XFER_CONTROL,
  55         .wMaxPacketSize  = cpu_to_le16(CTRL_PAYLOAD_MAX),
  56 };
  57
  58 /**
  59  * hw_ep_bit: calculates the bit number
  60  * @num: endpoint number
  61  * @dir: endpoint direction
  62  *
  63  * This function returns bit number
  64  */
  65 static inline int hw_ep_bit(int num, int dir)
  66 {
  67         return num + (dir ? 16 : 0);
  68 }
  69
  70 static inline int ep_to_bit(struct ci13xxx *ci, int n)
  71 {
  72         int fill = 16 - ci->hw_ep_max / 2;
  73
  74         if (n >= ci->hw_ep_max / 2)
  75                 n += fill;
  76
  77         return n;
  78 }
  79
  80 /**
  81  * hw_device_state: enables/disables interrupts & starts/stops device (execute
  82  *                  without interruption)
  83  * @dma: 0 => disable, !0 => enable and set dma engine
  84  *
  85  * This function returns an error code
  86  */
  87 static int hw_device_state(struct ci13xxx *ci, u32 dma)
  88 {
  89         if (dma) {
  90                 hw_write(ci, OP_ENDPTLISTADDR, ~0, dma);
  91                 /* interrupt, error, port change, reset, sleep/suspend */
  92                 hw_write(ci, OP_USBINTR, ~0,
  93                              USBi_UI|USBi_UEI|USBi_PCI|USBi_URI|USBi_SLI);
  94                 hw_write(ci, OP_USBCMD, USBCMD_RS, USBCMD_RS);
  95         } else {
  96                 hw_write(ci, OP_USBCMD, USBCMD_RS, 0);
  97                 hw_write(ci, OP_USBINTR, ~0, 0);
  98         }
  99         return 0;
 100 }
 101
 102 /**
 103  * hw_ep_flush: flush endpoint fifo (execute without interruption)
 104  * @num: endpoint number
 105  * @dir: endpoint direction
 106  *
 107  * This function returns an error code
 108  */
 109 static int hw_ep_flush(struct ci13xxx *ci, int num, int dir)
 110 {
 111         int n = hw_ep_bit(num, dir);
 112
 113         do {
 114                 /* flush any pending transfer */
 115                 hw_write(ci, OP_ENDPTFLUSH, BIT(n), BIT(n));
 116                 while (hw_read(ci, OP_ENDPTFLUSH, BIT(n)))
 117                         cpu_relax();
 118         } while (hw_read(ci, OP_ENDPTSTAT, BIT(n)));
 119
 120         return 0;
 121 }
 122
 123 /**
 124  * hw_ep_disable: disables endpoint (execute without interruption)
 125  * @num: endpoint number
 126  * @dir: endpoint direction
 127  *
 128  * This function returns an error code
 129  */
 130 static int hw_ep_disable(struct ci13xxx *ci, int num, int dir)
 131 {
 132         hw_ep_flush(ci, num, dir);
 133         hw_write(ci, OP_ENDPTCTRL + num,
 134                  dir ? ENDPTCTRL_TXE : ENDPTCTRL_RXE, 0);
 135         return 0;
 136 }
 137
 138 /**
 139  * hw_ep_enable: enables endpoint (execute without interruption)
 140  * @num:  endpoint number
 141  * @dir:  endpoint direction
 142  * @type: endpoint type
 143  *
 144  * This function returns an error code
 145  */
 146 static int hw_ep_enable(struct ci13xxx *ci, int num, int dir, int type)
 147 {
 148         u32 mask, data;
 149
 150         if (dir) {
 151                 mask  = ENDPTCTRL_TXT;  /* type    */
 152                 data  = type << ffs_nr(mask);
 153
 154                 mask |= ENDPTCTRL_TXS;  /* unstall */
 155                 mask |= ENDPTCTRL_TXR;  /* reset data toggle */
 156                 data |= ENDPTCTRL_TXR;
 157                 mask |= ENDPTCTRL_TXE;  /* enable  */
 158                 data |= ENDPTCTRL_TXE;
 159         } else {
 160                 mask  = ENDPTCTRL_RXT;  /* type    */
 161                 data  = type << ffs_nr(mask);
 162
 163                 mask |= ENDPTCTRL_RXS;  /* unstall */
 164                 mask |= ENDPTCTRL_RXR;  /* reset data toggle */
 165                 data |= ENDPTCTRL_RXR;
 166                 mask |= ENDPTCTRL_RXE;  /* enable  */
 167                 data |= ENDPTCTRL_RXE;
 168         }
 169         hw_write(ci, OP_ENDPTCTRL + num, mask, data);
 170         return 0;
 171 }
 172
 173 /**
 174  * hw_ep_get_halt: return endpoint halt status
 175  * @num: endpoint number
 176  * @dir: endpoint direction
 177  *
 178  * This function returns 1 if endpoint halted
 179  */
 180 static int hw_ep_get_halt(struct ci13xxx *ci, int num, int dir)
 181 {
 182         u32 mask = dir ? ENDPTCTRL_TXS : ENDPTCTRL_RXS;
 183
 184         return hw_read(ci, OP_ENDPTCTRL + num, mask) ? 1 : 0;
 185 }
 186
 187 /**
 188  * hw_test_and_clear_setup_status: test & clear setup status (execute without
 189  *                                 interruption)
 190  * @n: endpoint number
 191  *
 192  * This function returns setup status
 193  */
 194 static int hw_test_and_clear_setup_status(struct ci13xxx *ci, int n)
 195 {
 196         n = ep_to_bit(ci, n);
 197         return hw_test_and_clear(ci, OP_ENDPTSETUPSTAT, BIT(n));
 198 }
 199
 200 /**
 201  * hw_ep_prime: primes endpoint (execute without interruption)
 202  * @num:     endpoint number
 203  * @dir:     endpoint direction
 204  * @is_ctrl: true if control endpoint
 205  *
 206  * This function returns an error code
 207  */
 208 static int hw_ep_prime(struct ci13xxx *ci, int num, int dir, int is_ctrl)
 209 {
 210         int n = hw_ep_bit(num, dir);
 211
 212         if (is_ctrl && dir == RX && hw_read(ci, OP_ENDPTSETUPSTAT, BIT(num)))
 213                 return -EAGAIN;
 214
 215         hw_write(ci, OP_ENDPTPRIME, BIT(n), BIT(n));
 216
 217         while (hw_read(ci, OP_ENDPTPRIME, BIT(n)))
 218                 cpu_relax();
 219         if (is_ctrl && dir == RX && hw_read(ci, OP_ENDPTSETUPSTAT, BIT(num)))
 220                 return -EAGAIN;
 221
 222         /* status shoult be tested according with manual but it doesn't work */
 223         return 0;
 224 }
 225
 226 /**
 227  * hw_ep_set_halt: configures ep halt & resets data toggle after clear (execute
 228  *                 without interruption)
 229  * @num:   endpoint number
 230  * @dir:   endpoint direction
 231  * @value: true => stall, false => unstall
 232  *
 233  * This function returns an error code
 234  */
 235 static int hw_ep_set_halt(struct ci13xxx *ci, int num, int dir, int value)
 236 {
 237         if (value != 0 && value != 1)
 238                 return -EINVAL;
 239
 240         do {
 241                 enum ci13xxx_regs reg = OP_ENDPTCTRL + num;
 242                 u32 mask_xs = dir ? ENDPTCTRL_TXS : ENDPTCTRL_RXS;
 243                 u32 mask_xr = dir ? ENDPTCTRL_TXR : ENDPTCTRL_RXR;
 244
 245                 /* data toggle - reserved for EP0 but it's in ESS */
 246                 hw_write(ci, reg, mask_xs|mask_xr,
 247                           value ? mask_xs : mask_xr);
 248         } while (value != hw_ep_get_halt(ci, num, dir));
 249
 250         return 0;
 251 }
 252
 253 /**
 254  * hw_is_port_high_speed: test if port is high speed
 255  *
 256  * This function returns true if high speed port
 257  */
 258 static int hw_port_is_high_speed(struct ci13xxx *ci)
 259 {
 260         return ci->hw_bank.lpm ? hw_read(ci, OP_DEVLC, DEVLC_PSPD) :
 261                 hw_read(ci, OP_PORTSC, PORTSC_HSP);
 262 }
 263
 264 /**
 265  * hw_read_intr_enable: returns interrupt enable register
 266  *
 267  * This function returns register data
 268  */
 269 static u32 hw_read_intr_enable(struct ci13xxx *ci)
 270 {
 271         return hw_read(ci, OP_USBINTR, ~0);
 272 }
 273
 274 /**
 275  * hw_read_intr_status: returns interrupt status register
 276  *
 277  * This function returns register data
 278  */
 279 static u32 hw_read_intr_status(struct ci13xxx *ci)
 280 {
 281         return hw_read(ci, OP_USBSTS, ~0);
 282 }
 283
 284 /**
 285  * hw_test_and_clear_complete: test & clear complete status (execute without
 286  *                             interruption)
 287  * @n: endpoint number
 288  *
 289  * This function returns complete status
 290  */
 291 static int hw_test_and_clear_complete(struct ci13xxx *ci, int n)
 292 {
 293         n = ep_to_bit(ci, n);
 294         return hw_test_and_clear(ci, OP_ENDPTCOMPLETE, BIT(n));
 295 }
 296
 297 /**
 298  * hw_test_and_clear_intr_active: test & clear active interrupts (execute
 299  *                                without interruption)
 300  *
 301  * This function returns active interrutps
 302  */
 303 static u32 hw_test_and_clear_intr_active(struct ci13xxx *ci)
 304 {
 305         u32 reg = hw_read_intr_status(ci) & hw_read_intr_enable(ci);
 306
 307         hw_write(ci, OP_USBSTS, ~0, reg);
 308         return reg;
 309 }
 310
 311 /**
 312  * hw_test_and_clear_setup_guard: test & clear setup guard (execute without
 313  *                                interruption)
 314  *
 315  * This function returns guard value
 316  */
 317 static int hw_test_and_clear_setup_guard(struct ci13xxx *ci)
 318 {
 319         return hw_test_and_write(ci, OP_USBCMD, USBCMD_SUTW, 0);
 320 }
 321
 322 /**
 323  * hw_test_and_set_setup_guard: test & set setup guard (execute without
 324  *                              interruption)
 325  *
 326  * This function returns guard value
 327  */
 328 static int hw_test_and_set_setup_guard(struct ci13xxx *ci)
 329 {
 330         return hw_test_and_write(ci, OP_USBCMD, USBCMD_SUTW, USBCMD_SUTW);
 331 }
 332
 333 /**
 334  * hw_usb_set_address: configures USB address (execute without interruption)
 335  * @value: new USB address
 336  *
 337  * This function explicitly sets the address, without the "USBADRA" (advance)
 338  * feature, which is not supported by older versions of the controller.
 339  */
 340 static void hw_usb_set_address(struct ci13xxx *ci, u8 value)
 341 {
 342         hw_write(ci, OP_DEVICEADDR, DEVICEADDR_USBADR,
 343                  value << ffs_nr(DEVICEADDR_USBADR));
 344 }
 345
 346 /**
 347  * hw_usb_reset: restart device after a bus reset (execute without
 348  *               interruption)
 349  *
 350  * This function returns an error code
 351  */
 352 static int hw_usb_reset(struct ci13xxx *ci)
 353 {
 354         hw_usb_set_address(ci, 0);
 355
 356         /* ESS flushes only at end?!? */
 357         hw_write(ci, OP_ENDPTFLUSH,    ~0, ~0);
 358
 359         /* clear setup token semaphores */
 360         hw_write(ci, OP_ENDPTSETUPSTAT, 0,  0);
 361
 362         /* clear complete status */
 363         hw_write(ci, OP_ENDPTCOMPLETE,  0,  0);
 364
 365         /* wait until all bits cleared */
 366         while (hw_read(ci, OP_ENDPTPRIME, ~0))
 367                 udelay(10);             /* not RTOS friendly */
 368
 369         /* reset all endpoints ? */
 370
 371         /* reset internal status and wait for further instructions
 372            no need to verify the port reset status (ESS does it) */
 373
 374         return 0;
 375 }
 376
 377 /******************************************************************************
 378  * UTIL block
 379  *****************************************************************************/
 380 /**
 381  * _usb_addr: calculates endpoint address from direction & number
 382  * @ep:  endpoint
 383  */
 384 static inline u8 _usb_addr(struct ci13xxx_ep *ep)
 385 {
 386         return ((ep->dir == TX) ? USB_ENDPOINT_DIR_MASK : 0) | ep->num;
 387 }
 388
 389 /**
 390  * _hardware_queue: configures a request at hardware level
 391  * @gadget: gadget
 392  * @mEp:    endpoint
 393  *
 394  * This function returns an error code
 395  */
 396 static int _hardware_enqueue(struct ci13xxx_ep *mEp, struct ci13xxx_req *mReq)
 397 {
 398         struct ci13xxx *ci = mEp->ci;
 399         unsigned i;
 400         int ret = 0;
 401         unsigned length = mReq->req.length;
 402
 403         /* don't queue twice */
 404         if (mReq->req.status == -EALREADY)
 405                 return -EALREADY;
 406
 407         mReq->req.status = -EALREADY;
 408
 409         if (mReq->req.zero && length && (length % mEp->ep.maxpacket == 0)) {
 410                 mReq->zptr = dma_pool_alloc(mEp->td_pool, GFP_ATOMIC,
 411                                            &mReq->zdma);
 412                 if (mReq->zptr == NULL)
 413                         return -ENOMEM;
 414
 415                 memset(mReq->zptr, 0, sizeof(*mReq->zptr));
 416                 mReq->zptr->next    = TD_TERMINATE;
 417                 mReq->zptr->token   = TD_STATUS_ACTIVE;
 418                 if (!mReq->req.no_interrupt)
 419                         mReq->zptr->token   |= TD_IOC;
 420         }
 421         ret = usb_gadget_map_request(&ci->gadget, &mReq->req, mEp->dir);
 422         if (ret)
 423                 return ret;
 424
 425         /*
 426          * TD configuration
 427          * TODO - handle requests which spawns into several TDs
 428          */
 429         memset(mReq->ptr, 0, sizeof(*mReq->ptr));
 430         mReq->ptr->token    = length << ffs_nr(TD_TOTAL_BYTES);
 431         mReq->ptr->token   &= TD_TOTAL_BYTES;
 432         mReq->ptr->token   |= TD_STATUS_ACTIVE;
 433         if (mReq->zptr) {
 434                 mReq->ptr->next    = mReq->zdma;
 435         } else {
 436                 mReq->ptr->next    = TD_TERMINATE;
 437                 if (!mReq->req.no_interrupt)
 438                         mReq->ptr->token  |= TD_IOC;
 439         }
 440         mReq->ptr->page[0]  = mReq->req.dma;
 441         for (i = 1; i < 5; i++)
 442                 mReq->ptr->page[i] =
 443                         (mReq->req.dma + i * CI13XXX_PAGE_SIZE) & ~TD_RESERVED_MASK;
 444
 445         if (!list_empty(&mEp->qh.queue)) {
 446                 struct ci13xxx_req *mReqPrev;
 447                 int n = hw_ep_bit(mEp->num, mEp->dir);
 448                 int tmp_stat;
 449
 450                 mReqPrev = list_entry(mEp->qh.queue.prev,
 451                                 struct ci13xxx_req, queue);
 452                 if (mReqPrev->zptr)
 453                         mReqPrev->zptr->next = mReq->dma & TD_ADDR_MASK;
 454                 else
 455                         mReqPrev->ptr->next = mReq->dma & TD_ADDR_MASK;
 456                 wmb();
 457                 if (hw_read(ci, OP_ENDPTPRIME, BIT(n)))
 458                         goto done;
 459                 do {
 460                         hw_write(ci, OP_USBCMD, USBCMD_ATDTW, USBCMD_ATDTW);
 461                         tmp_stat = hw_read(ci, OP_ENDPTSTAT, BIT(n));
 462                 } while (!hw_read(ci, OP_USBCMD, USBCMD_ATDTW));
 463                 hw_write(ci, OP_USBCMD, USBCMD_ATDTW, 0);
 464                 if (tmp_stat)
 465                         goto done;
 466         }
 467
 468         /*  QH configuration */
 469         mEp->qh.ptr->td.next   = mReq->dma;    /* TERMINATE = 0 */
 470         mEp->qh.ptr->td.token &= ~TD_STATUS;   /* clear status */
 471         mEp->qh.ptr->cap |=  QH_ZLT;
 472
 473         wmb();   /* synchronize before ep prime */
 474
 475         ret = hw_ep_prime(ci, mEp->num, mEp->dir,
 476                            mEp->type == USB_ENDPOINT_XFER_CONTROL);
 477 done:
 478         return ret;
 479 }
 480
 481 /**
 482  * _hardware_dequeue: handles a request at hardware level
 483  * @gadget: gadget
 484  * @mEp:    endpoint
 485  *
 486  * This function returns an error code
 487  */
 488 static int _hardware_dequeue(struct ci13xxx_ep *mEp, struct ci13xxx_req *mReq)
 489 {
 490         if (mReq->req.status != -EALREADY)
 491                 return -EINVAL;
 492
 493         if ((TD_STATUS_ACTIVE & mReq->ptr->token) != 0)
 494                 return -EBUSY;
 495
 496         if (mReq->zptr) {
 497                 if ((TD_STATUS_ACTIVE & mReq->zptr->token) != 0)
 498                         return -EBUSY;
 499                 dma_pool_free(mEp->td_pool, mReq->zptr, mReq->zdma);
 500                 mReq->zptr = NULL;
 501         }
 502
 503         mReq->req.status = 0;
 504
 505         usb_gadget_unmap_request(&mEp->ci->gadget, &mReq->req, mEp->dir);
 506
 507         mReq->req.status = mReq->ptr->token & TD_STATUS;
 508         if ((TD_STATUS_HALTED & mReq->req.status) != 0)
 509                 mReq->req.status = -1;
 510         else if ((TD_STATUS_DT_ERR & mReq->req.status) != 0)
 511                 mReq->req.status = -1;
 512         else if ((TD_STATUS_TR_ERR & mReq->req.status) != 0)
 513                 mReq->req.status = -1;
 514
 515         mReq->req.actual   = mReq->ptr->token & TD_TOTAL_BYTES;
 516         mReq->req.actual >>= ffs_nr(TD_TOTAL_BYTES);
 517         mReq->req.actual   = mReq->req.length - mReq->req.actual;
 518         mReq->req.actual   = mReq->req.status ? 0 : mReq->req.actual;
 519
 520         return mReq->req.actual;
 521 }
 522
 523 /**
 524  * _ep_nuke: dequeues all endpoint requests
 525  * @mEp: endpoint
 526  *
 527  * This function returns an error code
 528  * Caller must hold lock
 529  */
 530 static int _ep_nuke(struct ci13xxx_ep *mEp)
 531 __releases(mEp->lock)
 532 __acquires(mEp->lock)
 533 {
 534         if (mEp == NULL)
 535                 return -EINVAL;
 536
 537         hw_ep_flush(mEp->ci, mEp->num, mEp->dir);
 538
 539         while (!list_empty(&mEp->qh.queue)) {
 540
 541                 /* pop oldest request */
 542                 struct ci13xxx_req *mReq = \
 543                         list_entry(mEp->qh.queue.next,
 544                                    struct ci13xxx_req, queue);
 545                 list_del_init(&mReq->queue);
 546                 mReq->req.status = -ESHUTDOWN;
 547
 548                 if (mReq->req.complete != NULL) {
 549                         spin_unlock(mEp->lock);
 550                         mReq->req.complete(&mEp->ep, &mReq->req);
 551                         spin_lock(mEp->lock);
 552                 }
 553         }
 554         return 0;
 555 }
 556
 557 /**
 558  * _gadget_stop_activity: stops all USB activity, flushes & disables all endpts
 559  * @gadget: gadget
 560  *
 561  * This function returns an error code
 562  */
 563 static int _gadget_stop_activity(struct usb_gadget *gadget)
 564 {
 565         struct usb_ep *ep;
 566         struct ci13xxx    *ci = container_of(gadget, struct ci13xxx, gadget);
 567         unsigned long flags;
 568
 569         spin_lock_irqsave(&ci->lock, flags);
 570         ci->gadget.speed = USB_SPEED_UNKNOWN;
 571         ci->remote_wakeup = 0;
 572         ci->suspended = 0;
 573         spin_unlock_irqrestore(&ci->lock, flags);
 574
 575         /* flush all endpoints */
 576         gadget_for_each_ep(ep, gadget) {
 577                 usb_ep_fifo_flush(ep);
 578         }
 579         usb_ep_fifo_flush(&ci->ep0out->ep);
 580         usb_ep_fifo_flush(&ci->ep0in->ep);
 581
 582         if (ci->driver)
 583                 ci->driver->disconnect(gadget);
 584
 585         /* make sure to disable all endpoints */
 586         gadget_for_each_ep(ep, gadget) {
 587                 usb_ep_disable(ep);
 588         }
 589
 590         if (ci->status != NULL) {
 591                 usb_ep_free_request(&ci->ep0in->ep, ci->status);
 592                 ci->status = NULL;
 593         }
 594
 595         return 0;
 596 }
 597
 598 /******************************************************************************
 599  * ISR block
 600  *****************************************************************************/
 601 /**
 602  * isr_reset_handler: USB reset interrupt handler
 603  * @ci: UDC device
 604  *
 605  * This function resets USB engine after a bus reset occurred
 606  */
 607 static void isr_reset_handler(struct ci13xxx *ci)
 608 __releases(ci->lock)
 609 __acquires(ci->lock)
 610 {
 611         int retval;
 612
 613         dbg_event(0xFF, "BUS RST", 0);
 614
 615         spin_unlock(&ci->lock);
 616         retval = _gadget_stop_activity(&ci->gadget);
 617         if (retval)
 618                 goto done;
 619
 620         retval = hw_usb_reset(ci);
 621         if (retval)
 622                 goto done;
 623
 624         ci->status = usb_ep_alloc_request(&ci->ep0in->ep, GFP_ATOMIC);
 625         if (ci->status == NULL)
 626                 retval = -ENOMEM;
 627
 628 done:
 629         spin_lock(&ci->lock);
 630
 631         if (retval)
 632                 dev_err(ci->dev, "error: %i\n", retval);
 633 }
 634
 635 /**
 636  * isr_get_status_complete: get_status request complete function
 637  * @ep:  endpoint
 638  * @req: request handled
 639  *
 640  * Caller must release lock
 641  */
 642 static void isr_get_status_complete(struct usb_ep *ep, struct usb_request *req)
 643 {
 644         if (ep == NULL || req == NULL)
 645                 return;
 646
 647         kfree(req->buf);
 648         usb_ep_free_request(ep, req);
 649 }
 650
 651 /**
 652  * isr_get_status_response: get_status request response
 653  * @ci: ci struct
 654  * @setup: setup request packet
 655  *
 656  * This function returns an error code
 657  */
 658 static int isr_get_status_response(struct ci13xxx *ci,
 659                                    struct usb_ctrlrequest *setup)
 660 __releases(mEp->lock)
 661 __acquires(mEp->lock)
 662 {
 663         struct ci13xxx_ep *mEp = ci->ep0in;
 664         struct usb_request *req = NULL;
 665         gfp_t gfp_flags = GFP_ATOMIC;
 666         int dir, num, retval;
 667
 668         if (mEp == NULL || setup == NULL)
 669                 return -EINVAL;
 670
 671         spin_unlock(mEp->lock);
 672         req = usb_ep_alloc_request(&mEp->ep, gfp_flags);
 673         spin_lock(mEp->lock);
 674         if (req == NULL)
 675                 return -ENOMEM;
 676
 677         req->complete = isr_get_status_complete;
 678         req->length   = 2;
 679         req->buf      = kzalloc(req->length, gfp_flags);
 680         if (req->buf == NULL) {
 681                 retval = -ENOMEM;
 682                 goto err_free_req;
 683         }
 684
 685         if ((setup->bRequestType & USB_RECIP_MASK) == USB_RECIP_DEVICE) {
 686                 /* Assume that device is bus powered for now. */
 687                 *(u16 *)req->buf = ci->remote_wakeup << 1;
 688                 retval = 0;
 689         } else if ((setup->bRequestType & USB_RECIP_MASK) \
 690                    == USB_RECIP_ENDPOINT) {
 691                 dir = (le16_to_cpu(setup->wIndex) & USB_ENDPOINT_DIR_MASK) ?
 692                         TX : RX;
 693                 num =  le16_to_cpu(setup->wIndex) & USB_ENDPOINT_NUMBER_MASK;
 694                 *(u16 *)req->buf = hw_ep_get_halt(ci, num, dir);
 695         }
 696         /* else do nothing; reserved for future use */
 697
 698         spin_unlock(mEp->lock);
 699         retval = usb_ep_queue(&mEp->ep, req, gfp_flags);
 700         spin_lock(mEp->lock);
 701         if (retval)
 702                 goto err_free_buf;
 703
 704         return 0;
 705
 706  err_free_buf:
 707         kfree(req->buf);
 708  err_free_req:
 709         spin_unlock(mEp->lock);
 710         usb_ep_free_request(&mEp->ep, req);
 711         spin_lock(mEp->lock);
 712         return retval;
 713 }
 714
 715 /**
 716  * isr_setup_status_complete: setup_status request complete function
 717  * @ep:  endpoint
 718  * @req: request handled
 719  *
 720  * Caller must release lock. Put the port in test mode if test mode
 721  * feature is selected.
 722  */
 723 static void
 724 isr_setup_status_complete(struct usb_ep *ep, struct usb_request *req)
 725 {
 726         struct ci13xxx *ci = req->context;
 727         unsigned long flags;
 728
 729         if (ci->setaddr) {
 730                 hw_usb_set_address(ci, ci->address);
 731                 ci->setaddr = false;
 732         }
 733
 734         spin_lock_irqsave(&ci->lock, flags);
 735         if (ci->test_mode)
 736                 hw_port_test_set(ci, ci->test_mode);
 737         spin_unlock_irqrestore(&ci->lock, flags);
 738 }
 739
 740 /**
 741  * isr_setup_status_phase: queues the status phase of a setup transation
 742  * @ci: ci struct
 743  *
 744  * This function returns an error code
 745  */
 746 static int isr_setup_status_phase(struct ci13xxx *ci)
 747 __releases(mEp->lock)
 748 __acquires(mEp->lock)
 749 {
 750         int retval;
 751         struct ci13xxx_ep *mEp;
 752
 753         mEp = (ci->ep0_dir == TX) ? ci->ep0out : ci->ep0in;
 754         ci->status->context = ci;
 755         ci->status->complete = isr_setup_status_complete;
 756
 757         spin_unlock(mEp->lock);
 758         retval = usb_ep_queue(&mEp->ep, ci->status, GFP_ATOMIC);
 759         spin_lock(mEp->lock);
 760
 761         return retval;
 762 }
 763
 764 /**
 765  * isr_tr_complete_low: transaction complete low level handler
 766  * @mEp: endpoint
 767  *
 768  * This function returns an error code
 769  * Caller must hold lock
 770  */
 771 static int isr_tr_complete_low(struct ci13xxx_ep *mEp)
 772 __releases(mEp->lock)
 773 __acquires(mEp->lock)
 774 {
 775         struct ci13xxx_req *mReq, *mReqTemp;
 776         struct ci13xxx_ep *mEpTemp = mEp;
 777         int uninitialized_var(retval);
 778
 779         if (list_empty(&mEp->qh.queue))
 780                 return -EINVAL;
 781
 782         list_for_each_entry_safe(mReq, mReqTemp, &mEp->qh.queue,
 783                         queue) {
 784                 retval = _hardware_dequeue(mEp, mReq);
 785                 if (retval < 0)
 786                         break;
 787                 list_del_init(&mReq->queue);
 788                 dbg_done(_usb_addr(mEp), mReq->ptr->token, retval);
 789                 if (mReq->req.complete != NULL) {
 790                         spin_unlock(mEp->lock);
 791                         if ((mEp->type == USB_ENDPOINT_XFER_CONTROL) &&
 792                                         mReq->req.length)
 793                                 mEpTemp = mEp->ci->ep0in;
 794                         mReq->req.complete(&mEpTemp->ep, &mReq->req);
 795                         spin_lock(mEp->lock);
 796                 }
 797         }
 798
 799         if (retval == -EBUSY)
 800                 retval = 0;
 801         if (retval < 0)
 802                 dbg_event(_usb_addr(mEp), "DONE", retval);
 803
 804         return retval;
 805 }
 806
 807 /**
 808  * isr_tr_complete_handler: transaction complete interrupt handler
 809  * @ci: UDC descriptor
 810  *
 811  * This function handles traffic events
 812  */
 813 static void isr_tr_complete_handler(struct ci13xxx *ci)
 814 __releases(ci->lock)
 815 __acquires(ci->lock)
 816 {
 817         unsigned i;
 818         u8 tmode = 0;
 819
 820         for (i = 0; i < ci->hw_ep_max; i++) {
 821                 struct ci13xxx_ep *mEp  = &ci->ci13xxx_ep[i];
 822                 int type, num, dir, err = -EINVAL;
 823                 struct usb_ctrlrequest req;
 824
 825                 if (mEp->ep.desc == NULL)
 826                         continue;   /* not configured */
 827
 828                 if (hw_test_and_clear_complete(ci, i)) {
 829                         err = isr_tr_complete_low(mEp);
 830                         if (mEp->type == USB_ENDPOINT_XFER_CONTROL) {
 831                                 if (err > 0)   /* needs status phase */
 832                                         err = isr_setup_status_phase(ci);
 833                                 if (err < 0) {
 834                                         dbg_event(_usb_addr(mEp),
 835                                                   "ERROR", err);
 836                                         spin_unlock(&ci->lock);
 837                                         if (usb_ep_set_halt(&mEp->ep))
 838                                                 dev_err(ci->dev,
 839                                                         "error: ep_set_halt\n");
 840                                         spin_lock(&ci->lock);
 841                                 }
 842                         }
 843                 }
 844
 845                 if (mEp->type != USB_ENDPOINT_XFER_CONTROL ||
 846                     !hw_test_and_clear_setup_status(ci, i))
 847                         continue;
 848
 849                 if (i != 0) {
 850                         dev_warn(ci->dev, "ctrl traffic at endpoint %d\n", i);
 851                         continue;
 852                 }
 853
 854                 /*
 855                  * Flush data and handshake transactions of previous
 856                  * setup packet.
 857                  */
 858                 _ep_nuke(ci->ep0out);
 859                 _ep_nuke(ci->ep0in);
 860
 861                 /* read_setup_packet */
 862                 do {
 863                         hw_test_and_set_setup_guard(ci);
 864                         memcpy(&req, &mEp->qh.ptr->setup, sizeof(req));
 865                 } while (!hw_test_and_clear_setup_guard(ci));
 866
 867                 type = req.bRequestType;
 868
 869                 ci->ep0_dir = (type & USB_DIR_IN) ? TX : RX;
 870
 871                 dbg_setup(_usb_addr(mEp), &req);
 872
 873                 switch (req.bRequest) {
 874                 case USB_REQ_CLEAR_FEATURE:
 875                         if (type == (USB_DIR_OUT|USB_RECIP_ENDPOINT) &&
 876                                         le16_to_cpu(req.wValue) ==
 877                                         USB_ENDPOINT_HALT) {
 878                                 if (req.wLength != 0)
 879                                         break;
 880                                 num  = le16_to_cpu(req.wIndex);
 881                                 dir = num & USB_ENDPOINT_DIR_MASK;
 882                                 num &= USB_ENDPOINT_NUMBER_MASK;
 883                                 if (dir) /* TX */
 884                                         num += ci->hw_ep_max/2;
 885                                 if (!ci->ci13xxx_ep[num].wedge) {
 886                                         spin_unlock(&ci->lock);
 887                                         err = usb_ep_clear_halt(
 888                                                 &ci->ci13xxx_ep[num].ep);
 889                                         spin_lock(&ci->lock);
 890                                         if (err)
 891                                                 break;
 892                                 }
 893                                 err = isr_setup_status_phase(ci);
 894                         } else if (type == (USB_DIR_OUT|USB_RECIP_DEVICE) &&
 895                                         le16_to_cpu(req.wValue) ==
 896                                         USB_DEVICE_REMOTE_WAKEUP) {
 897                                 if (req.wLength != 0)
 898                                         break;
 899                                 ci->remote_wakeup = 0;
 900                                 err = isr_setup_status_phase(ci);
 901                         } else {
 902                                 goto delegate;
 903                         }
 904                         break;
 905                 case USB_REQ_GET_STATUS:
 906                         if (type != (USB_DIR_IN|USB_RECIP_DEVICE)   &&
 907                             type != (USB_DIR_IN|USB_RECIP_ENDPOINT) &&
 908                             type != (USB_DIR_IN|USB_RECIP_INTERFACE))
 909                                 goto delegate;
 910                         if (le16_to_cpu(req.wLength) != 2 ||
 911                             le16_to_cpu(req.wValue)  != 0)
 912                                 break;
 913                         err = isr_get_status_response(ci, &req);
 914                         break;
 915                 case USB_REQ_SET_ADDRESS:
 916                         if (type != (USB_DIR_OUT|USB_RECIP_DEVICE))
 917                                 goto delegate;
 918                         if (le16_to_cpu(req.wLength) != 0 ||
 919                             le16_to_cpu(req.wIndex)  != 0)
 920                                 break;
 921                         ci->address = (u8)le16_to_cpu(req.wValue);
 922                         ci->setaddr = true;
 923                         err = isr_setup_status_phase(ci);
 924                         break;
 925                 case USB_REQ_SET_FEATURE:
 926                         if (type == (USB_DIR_OUT|USB_RECIP_ENDPOINT) &&
 927                                         le16_to_cpu(req.wValue) ==
 928                                         USB_ENDPOINT_HALT) {
 929                                 if (req.wLength != 0)
 930                                         break;
 931                                 num  = le16_to_cpu(req.wIndex);
 932                                 dir = num & USB_ENDPOINT_DIR_MASK;
 933                                 num &= USB_ENDPOINT_NUMBER_MASK;
 934                                 if (dir) /* TX */
 935                                         num += ci->hw_ep_max/2;
 936
 937                                 spin_unlock(&ci->lock);
 938                                 err = usb_ep_set_halt(&ci->ci13xxx_ep[num].ep);
 939                                 spin_lock(&ci->lock);
 940                                 if (!err)
 941                                         isr_setup_status_phase(ci);
 942                         } else if (type == (USB_DIR_OUT|USB_RECIP_DEVICE)) {
 943                                 if (req.wLength != 0)
 944                                         break;
 945                                 switch (le16_to_cpu(req.wValue)) {
 946                                 case USB_DEVICE_REMOTE_WAKEUP:
 947                                         ci->remote_wakeup = 1;
 948                                         err = isr_setup_status_phase(ci);
 949                                         break;
 950                                 case USB_DEVICE_TEST_MODE:
 951                                         tmode = le16_to_cpu(req.wIndex) >> 8;
 952                                         switch (tmode) {
 953                                         case TEST_J:
 954                                         case TEST_K:
 955                                         case TEST_SE0_NAK:
 956                                         case TEST_PACKET:
 957                                         case TEST_FORCE_EN:
 958                                                 ci->test_mode = tmode;
 959                                                 err = isr_setup_status_phase(
 960                                                                 ci);
 961                                                 break;
 962                                         default:
 963                                                 break;
 964                                         }
 965                                 default:
 966                                         goto delegate;
 967                                 }
 968                         } else {
 969                                 goto delegate;
 970                         }
 971                         break;
 972                 default:
 973 delegate:
 974                         if (req.wLength == 0)   /* no data phase */
 975                                 ci->ep0_dir = TX;
 976
 977                         spin_unlock(&ci->lock);
 978                         err = ci->driver->setup(&ci->gadget, &req);
 979                         spin_lock(&ci->lock);
 980                         break;
 981                 }
 982
 983                 if (err < 0) {
 984                         dbg_event(_usb_addr(mEp), "ERROR", err);
 985
 986                         spin_unlock(&ci->lock);
 987                         if (usb_ep_set_halt(&mEp->ep))
 988                                 dev_err(ci->dev, "error: ep_set_halt\n");
 989                         spin_lock(&ci->lock);
 990                 }
 991         }
 992 }
 993
 994 /******************************************************************************
 995  * ENDPT block
 996  *****************************************************************************/
 997 /**
 998  * ep_enable: configure endpoint, making it usable
 999  *
1000  * Check usb_ep_enable() at "usb_gadget.h" for details
1001  */
1002 static int ep_enable(struct usb_ep *ep,
1003                      const struct usb_endpoint_descriptor *desc)
1004 {
1005         struct ci13xxx_ep *mEp = container_of(ep, struct ci13xxx_ep, ep);
1006         int retval = 0;
1007         unsigned long flags;
1008
1009         if (ep == NULL || desc == NULL)
1010                 return -EINVAL;
1011
1012         spin_lock_irqsave(mEp->lock, flags);
1013
1014         /* only internal SW should enable ctrl endpts */
1015
1016         mEp->ep.desc = desc;
1017
1018         if (!list_empty(&mEp->qh.queue))
1019                 dev_warn(mEp->ci->dev, "enabling a non-empty endpoint!\n");
1020
1021         mEp->dir  = usb_endpoint_dir_in(desc) ? TX : RX;
1022         mEp->num  = usb_endpoint_num(desc);
1023         mEp->type = usb_endpoint_type(desc);
1024
1025         mEp->ep.maxpacket = usb_endpoint_maxp(desc);
1026
1027         dbg_event(_usb_addr(mEp), "ENABLE", 0);
1028
1029         mEp->qh.ptr->cap = 0;
1030
1031         if (mEp->type == USB_ENDPOINT_XFER_CONTROL)
1032                 mEp->qh.ptr->cap |=  QH_IOS;
1033         else if (mEp->type == USB_ENDPOINT_XFER_ISOC)
1034                 mEp->qh.ptr->cap &= ~QH_MULT;
1035         else
1036                 mEp->qh.ptr->cap &= ~QH_ZLT;
1037
1038         mEp->qh.ptr->cap |=
1039                 (mEp->ep.maxpacket << ffs_nr(QH_MAX_PKT)) & QH_MAX_PKT;
1040         mEp->qh.ptr->td.next |= TD_TERMINATE;   /* needed? */
1041
1042         /*
1043          * Enable endpoints in the HW other than ep0 as ep0
1044          * is always enabled
1045          */
1046         if (mEp->num)
1047                 retval |= hw_ep_enable(mEp->ci, mEp->num, mEp->dir, mEp->type);
1048
1049         spin_unlock_irqrestore(mEp->lock, flags);
1050         return retval;
1051 }
1052
1053 /**
1054  * ep_disable: endpoint is no longer usable
1055  *
1056  * Check usb_ep_disable() at "usb_gadget.h" for details
1057  */
1058 static int ep_disable(struct usb_ep *ep)
1059 {
1060         struct ci13xxx_ep *mEp = container_of(ep, struct ci13xxx_ep, ep);
1061         int direction, retval = 0;
1062         unsigned long flags;
1063
1064         if (ep == NULL)
1065                 return -EINVAL;
1066         else if (mEp->ep.desc == NULL)
1067                 return -EBUSY;
1068
1069         spin_lock_irqsave(mEp->lock, flags);
1070
1071         /* only internal SW should disable ctrl endpts */
1072
1073         direction = mEp->dir;
1074         do {
1075                 dbg_event(_usb_addr(mEp), "DISABLE", 0);
1076
1077                 retval |= _ep_nuke(mEp);
1078                 retval |= hw_ep_disable(mEp->ci, mEp->num, mEp->dir);
1079
1080                 if (mEp->type == USB_ENDPOINT_XFER_CONTROL)
1081                         mEp->dir = (mEp->dir == TX) ? RX : TX;
1082
1083         } while (mEp->dir != direction);
1084
1085         mEp->ep.desc = NULL;
1086
1087         spin_unlock_irqrestore(mEp->lock, flags);
1088         return retval;
1089 }
1090
1091 /**
1092  * ep_alloc_request: allocate a request object to use with this endpoint
1093  *
1094  * Check usb_ep_alloc_request() at "usb_gadget.h" for details
1095  */
1096 static struct usb_request *ep_alloc_request(struct usb_ep *ep, gfp_t gfp_flags)
1097 {
1098         struct ci13xxx_ep  *mEp  = container_of(ep, struct ci13xxx_ep, ep);
1099         struct ci13xxx_req *mReq = NULL;
1100
1101         if (ep == NULL)
1102                 return NULL;
1103
1104         mReq = kzalloc(sizeof(struct ci13xxx_req), gfp_flags);
1105         if (mReq != NULL) {
1106                 INIT_LIST_HEAD(&mReq->queue);
1107
1108                 mReq->ptr = dma_pool_alloc(mEp->td_pool, gfp_flags,
1109                                            &mReq->dma);
1110                 if (mReq->ptr == NULL) {
1111                         kfree(mReq);
1112                         mReq = NULL;
1113                 }
1114         }
1115
1116         dbg_event(_usb_addr(mEp), "ALLOC", mReq == NULL);
1117
1118         return (mReq == NULL) ? NULL : &mReq->req;
1119 }
1120
1121 /**
1122  * ep_free_request: frees a request object
1123  *
1124  * Check usb_ep_free_request() at "usb_gadget.h" for details
1125  */
1126 static void ep_free_request(struct usb_ep *ep, struct usb_request *req)
1127 {
1128         struct ci13xxx_ep  *mEp  = container_of(ep,  struct ci13xxx_ep, ep);
1129         struct ci13xxx_req *mReq = container_of(req, struct ci13xxx_req, req);
1130         unsigned long flags;
1131
1132         if (ep == NULL || req == NULL) {
1133                 return;
1134         } else if (!list_empty(&mReq->queue)) {
1135                 dev_err(mEp->ci->dev, "freeing queued request\n");
1136                 return;
1137         }
1138
1139         spin_lock_irqsave(mEp->lock, flags);
1140
1141         if (mReq->ptr)
1142                 dma_pool_free(mEp->td_pool, mReq->ptr, mReq->dma);
1143         kfree(mReq);
1144
1145         dbg_event(_usb_addr(mEp), "FREE", 0);
1146
1147         spin_unlock_irqrestore(mEp->lock, flags);
1148 }
1149
1150 /**
1151  * ep_queue: queues (submits) an I/O request to an endpoint
1152  *
1153  * Check usb_ep_queue()* at usb_gadget.h" for details
1154  */
1155 static int ep_queue(struct usb_ep *ep, struct usb_request *req,
1156                     gfp_t __maybe_unused gfp_flags)
1157 {
1158         struct ci13xxx_ep  *mEp  = container_of(ep,  struct ci13xxx_ep, ep);
1159         struct ci13xxx_req *mReq = container_of(req, struct ci13xxx_req, req);
1160         struct ci13xxx *ci = mEp->ci;
1161         int retval = 0;
1162         unsigned long flags;
1163
1164         if (ep == NULL || req == NULL || mEp->ep.desc == NULL)
1165                 return -EINVAL;
1166
1167         spin_lock_irqsave(mEp->lock, flags);
1168
1169         if (mEp->type == USB_ENDPOINT_XFER_CONTROL) {
1170                 if (req->length)
1171                         mEp = (ci->ep0_dir == RX) ?
1172                                ci->ep0out : ci->ep0in;
1173                 if (!list_empty(&mEp->qh.queue)) {
1174                         _ep_nuke(mEp);
1175                         retval = -EOVERFLOW;
1176                         dev_warn(mEp->ci->dev, "endpoint ctrl %X nuked\n",
1177                                  _usb_addr(mEp));
1178                 }
1179         }
1180
1181         /* first nuke then test link, e.g. previous status has not sent */
1182         if (!list_empty(&mReq->queue)) {
1183                 retval = -EBUSY;
1184                 dev_err(mEp->ci->dev, "request already in queue\n");
1185                 goto done;
1186         }
1187
1188         if (req->length > 4 * CI13XXX_PAGE_SIZE) {
1189                 req->length = 4 * CI13XXX_PAGE_SIZE;
1190                 retval = -EMSGSIZE;
1191                 dev_warn(mEp->ci->dev, "request length truncated\n");
1192         }
1193
1194         dbg_queue(_usb_addr(mEp), req, retval);
1195
1196         /* push request */
1197         mReq->req.status = -EINPROGRESS;
1198         mReq->req.actual = 0;
1199
1200         retval = _hardware_enqueue(mEp, mReq);
1201
1202         if (retval == -EALREADY) {
1203                 dbg_event(_usb_addr(mEp), "QUEUE", retval);
1204                 retval = 0;
1205         }
1206         if (!retval)
1207                 list_add_tail(&mReq->queue, &mEp->qh.queue);
1208
1209  done:
1210         spin_unlock_irqrestore(mEp->lock, flags);
1211         return retval;
1212 }
1213
1214 /**
1215  * ep_dequeue: dequeues (cancels, unlinks) an I/O request from an endpoint
1216  *
1217  * Check usb_ep_dequeue() at "usb_gadget.h" for details
1218  */
1219 static int ep_dequeue(struct usb_ep *ep, struct usb_request *req)
1220 {
1221         struct ci13xxx_ep  *mEp  = container_of(ep,  struct ci13xxx_ep, ep);
1222         struct ci13xxx_req *mReq = container_of(req, struct ci13xxx_req, req);
1223         unsigned long flags;
1224
1225         if (ep == NULL || req == NULL || mReq->req.status != -EALREADY ||
1226                 mEp->ep.desc == NULL || list_empty(&mReq->queue) ||
1227                 list_empty(&mEp->qh.queue))
1228                 return -EINVAL;
1229
1230         spin_lock_irqsave(mEp->lock, flags);
1231
1232         dbg_event(_usb_addr(mEp), "DEQUEUE", 0);
1233
1234         hw_ep_flush(mEp->ci, mEp->num, mEp->dir);
1235
1236         /* pop request */
1237         list_del_init(&mReq->queue);
1238
1239         usb_gadget_unmap_request(&mEp->ci->gadget, req, mEp->dir);
1240
1241         req->status = -ECONNRESET;
1242
1243         if (mReq->req.complete != NULL) {
1244                 spin_unlock(mEp->lock);
1245                 mReq->req.complete(&mEp->ep, &mReq->req);
1246                 spin_lock(mEp->lock);
1247         }
1248
1249         spin_unlock_irqrestore(mEp->lock, flags);
1250         return 0;
1251 }
1252
1253 /**
1254  * ep_set_halt: sets the endpoint halt feature
1255  *
1256  * Check usb_ep_set_halt() at "usb_gadget.h" for details
1257  */
1258 static int ep_set_halt(struct usb_ep *ep, int value)
1259 {
1260         struct ci13xxx_ep *mEp = container_of(ep, struct ci13xxx_ep, ep);
1261         int direction, retval = 0;
1262         unsigned long flags;
1263
1264         if (ep == NULL || mEp->ep.desc == NULL)
1265                 return -EINVAL;
1266
1267         spin_lock_irqsave(mEp->lock, flags);
1268
1269 #ifndef STALL_IN
1270         /* g_file_storage MS compliant but g_zero fails chapter 9 compliance */
1271         if (value && mEp->type == USB_ENDPOINT_XFER_BULK && mEp->dir == TX &&
1272             !list_empty(&mEp->qh.queue)) {
1273                 spin_unlock_irqrestore(mEp->lock, flags);
1274                 return -EAGAIN;
1275         }
1276 #endif
1277
1278         direction = mEp->dir;
1279         do {
1280                 dbg_event(_usb_addr(mEp), "HALT", value);
1281                 retval |= hw_ep_set_halt(mEp->ci, mEp->num, mEp->dir, value);
1282
1283                 if (!value)
1284                         mEp->wedge = 0;
1285
1286                 if (mEp->type == USB_ENDPOINT_XFER_CONTROL)
1287                         mEp->dir = (mEp->dir == TX) ? RX : TX;
1288
1289         } while (mEp->dir != direction);
1290
1291         spin_unlock_irqrestore(mEp->lock, flags);
1292         return retval;
1293 }
1294
1295 /**
1296  * ep_set_wedge: sets the halt feature and ignores clear requests
1297  *
1298  * Check usb_ep_set_wedge() at "usb_gadget.h" for details
1299  */
1300 static int ep_set_wedge(struct usb_ep *ep)
1301 {
1302         struct ci13xxx_ep *mEp = container_of(ep, struct ci13xxx_ep, ep);
1303         unsigned long flags;
1304
1305         if (ep == NULL || mEp->ep.desc == NULL)
1306                 return -EINVAL;
1307
1308         spin_lock_irqsave(mEp->lock, flags);
1309
1310         dbg_event(_usb_addr(mEp), "WEDGE", 0);
1311         mEp->wedge = 1;
1312
1313         spin_unlock_irqrestore(mEp->lock, flags);
1314
1315         return usb_ep_set_halt(ep);
1316 }
1317
1318 /**
1319  * ep_fifo_flush: flushes contents of a fifo
1320  *
1321  * Check usb_ep_fifo_flush() at "usb_gadget.h" for details
1322  */
1323 static void ep_fifo_flush(struct usb_ep *ep)
1324 {
1325         struct ci13xxx_ep *mEp = container_of(ep, struct ci13xxx_ep, ep);
1326         unsigned long flags;
1327
1328         if (ep == NULL) {
1329                 dev_err(mEp->ci->dev, "%02X: -EINVAL\n", _usb_addr(mEp));
1330                 return;
1331         }
1332
1333         spin_lock_irqsave(mEp->lock, flags);
1334
1335         dbg_event(_usb_addr(mEp), "FFLUSH", 0);
1336         hw_ep_flush(mEp->ci, mEp->num, mEp->dir);
1337
1338         spin_unlock_irqrestore(mEp->lock, flags);
1339 }
1340
1341 /**
1342  * Endpoint-specific part of the API to the USB controller hardware
1343  * Check "usb_gadget.h" for details
1344  */
1345 static const struct usb_ep_ops usb_ep_ops = {
1346         .enable        = ep_enable,
1347         .disable       = ep_disable,
1348         .alloc_request = ep_alloc_request,
1349         .free_request  = ep_free_request,
1350         .queue         = ep_queue,
1351         .dequeue       = ep_dequeue,
1352         .set_halt      = ep_set_halt,
1353         .set_wedge     = ep_set_wedge,
1354         .fifo_flush    = ep_fifo_flush,
1355 };
1356
1357 /******************************************************************************
1358  * GADGET block
1359  *****************************************************************************/
1360 static int ci13xxx_vbus_session(struct usb_gadget *_gadget, int is_active)
1361 {
1362         struct ci13xxx *ci = container_of(_gadget, struct ci13xxx, gadget);
1363         unsigned long flags;
1364         int gadget_ready = 0;
1365
1366         if (!(ci->platdata->flags & CI13XXX_PULLUP_ON_VBUS))
1367                 return -EOPNOTSUPP;
1368
1369         spin_lock_irqsave(&ci->lock, flags);
1370         ci->vbus_active = is_active;
1371         if (ci->driver)
1372                 gadget_ready = 1;
1373         spin_unlock_irqrestore(&ci->lock, flags);
1374
1375         if (gadget_ready) {
1376                 if (is_active) {
1377                         pm_runtime_get_sync(&_gadget->dev);
1378                         hw_device_reset(ci, USBMODE_CM_DC);
1379                         hw_device_state(ci, ci->ep0out->qh.dma);
1380                 } else {
1381                         hw_device_state(ci, 0);
1382                         if (ci->platdata->notify_event)
1383                                 ci->platdata->notify_event(ci,
1384                                 CI13XXX_CONTROLLER_STOPPED_EVENT);
1385                         _gadget_stop_activity(&ci->gadget);
1386                         pm_runtime_put_sync(&_gadget->dev);
1387                 }
1388         }
1389
1390         return 0;
1391 }
1392
1393 static int ci13xxx_wakeup(struct usb_gadget *_gadget)
1394 {
1395         struct ci13xxx *ci = container_of(_gadget, struct ci13xxx, gadget);
1396         unsigned long flags;
1397         int ret = 0;
1398
1399         spin_lock_irqsave(&ci->lock, flags);
1400         if (!ci->remote_wakeup) {
1401                 ret = -EOPNOTSUPP;
1402                 goto out;
1403         }
1404         if (!hw_read(ci, OP_PORTSC, PORTSC_SUSP)) {
1405                 ret = -EINVAL;
1406                 goto out;
1407         }
1408         hw_write(ci, OP_PORTSC, PORTSC_FPR, PORTSC_FPR);
1409 out:
1410         spin_unlock_irqrestore(&ci->lock, flags);
1411         return ret;
1412 }
1413
1414 static int ci13xxx_vbus_draw(struct usb_gadget *_gadget, unsigned mA)
1415 {
1416         struct ci13xxx *ci = container_of(_gadget, struct ci13xxx, gadget);
1417
1418         if (ci->transceiver)
1419                 return usb_phy_set_power(ci->transceiver, mA);
1420         return -ENOTSUPP;
1421 }
1422
1423 static int ci13xxx_start(struct usb_gadget *gadget,
1424                          struct usb_gadget_driver *driver);
1425 static int ci13xxx_stop(struct usb_gadget *gadget,
1426                         struct usb_gadget_driver *driver);
1427 /**
1428  * Device operations part of the API to the USB controller hardware,
1429  * which don't involve endpoints (or i/o)
1430  * Check  "usb_gadget.h" for details
1431  */
1432 static const struct usb_gadget_ops usb_gadget_ops = {
1433         .vbus_session   = ci13xxx_vbus_session,
1434         .wakeup         = ci13xxx_wakeup,
1435         .vbus_draw      = ci13xxx_vbus_draw,
1436         .udc_start      = ci13xxx_start,
1437         .udc_stop       = ci13xxx_stop,
1438 };
1439
1440 static int init_eps(struct ci13xxx *ci)
1441 {
1442         int retval = 0, i, j;
1443
1444         for (i = 0; i < ci->hw_ep_max/2; i++)
1445                 for (j = RX; j <= TX; j++) {
1446                         int k = i + j * ci->hw_ep_max/2;
1447                         struct ci13xxx_ep *mEp = &ci->ci13xxx_ep[k];
1448
1449                         scnprintf(mEp->name, sizeof(mEp->name), "ep%i%s", i,
1450                                         (j == TX)  ? "in" : "out");
1451
1452                         mEp->ci          = ci;
1453                         mEp->lock         = &ci->lock;
1454                         mEp->td_pool      = ci->td_pool;
1455
1456                         mEp->ep.name      = mEp->name;
1457                         mEp->ep.ops       = &usb_ep_ops;
1458                         /*
1459                          * for ep0: maxP defined in desc, for other
1460                          * eps, maxP is set by epautoconfig() called
1461                          * by gadget layer
1462                          */
1463                         mEp->ep.maxpacket = (unsigned short)~0;
1464
1465                         INIT_LIST_HEAD(&mEp->qh.queue);
1466                         mEp->qh.ptr = dma_pool_alloc(ci->qh_pool, GFP_KERNEL,
1467                                                      &mEp->qh.dma);
1468                         if (mEp->qh.ptr == NULL)
1469                                 retval = -ENOMEM;
1470                         else
1471                                 memset(mEp->qh.ptr, 0, sizeof(*mEp->qh.ptr));
1472
1473                         /*
1474                          * set up shorthands for ep0 out and in endpoints,
1475                          * don't add to gadget's ep_list
1476                          */
1477                         if (i == 0) {
1478                                 if (j == RX)
1479                                         ci->ep0out = mEp;
1480                                 else
1481                                         ci->ep0in = mEp;
1482
1483                                 mEp->ep.maxpacket = CTRL_PAYLOAD_MAX;
1484                                 continue;
1485                         }
1486
1487                         list_add_tail(&mEp->ep.ep_list, &ci->gadget.ep_list);
1488                 }
1489
1490         return retval;
1491 }
1492
1493 /**
1494  * ci13xxx_start: register a gadget driver
1495  * @gadget: our gadget
1496  * @driver: the driver being registered
1497  *
1498  * Interrupts are enabled here.
1499  */
1500 static int ci13xxx_start(struct usb_gadget *gadget,
1501                          struct usb_gadget_driver *driver)
1502 {
1503         struct ci13xxx *ci = container_of(gadget, struct ci13xxx, gadget);
1504         unsigned long flags;
1505         int retval = -ENOMEM;
1506
1507         if (driver->disconnect == NULL)
1508                 return -EINVAL;
1509
1510
1511         ci->ep0out->ep.desc = &ctrl_endpt_out_desc;
1512         retval = usb_ep_enable(&ci->ep0out->ep);
1513         if (retval)
1514                 return retval;
1515
1516         ci->ep0in->ep.desc = &ctrl_endpt_in_desc;
1517         retval = usb_ep_enable(&ci->ep0in->ep);
1518         if (retval)
1519                 return retval;
1520         spin_lock_irqsave(&ci->lock, flags);
1521
1522         ci->driver = driver;
1523         pm_runtime_get_sync(&ci->gadget.dev);
1524         if (ci->platdata->flags & CI13XXX_PULLUP_ON_VBUS) {
1525                 if (ci->vbus_active) {
1526                         if (ci->platdata->flags & CI13XXX_REGS_SHARED)
1527                                 hw_device_reset(ci, USBMODE_CM_DC);
1528                 } else {
1529                         pm_runtime_put_sync(&ci->gadget.dev);
1530                         goto done;
1531                 }
1532         }
1533
1534         retval = hw_device_state(ci, ci->ep0out->qh.dma);
1535         if (retval)
1536                 pm_runtime_put_sync(&ci->gadget.dev);
1537
1538  done:
1539         spin_unlock_irqrestore(&ci->lock, flags);
1540         return retval;
1541 }
1542
1543 /**
1544  * ci13xxx_stop: unregister a gadget driver
1545  */
1546 static int ci13xxx_stop(struct usb_gadget *gadget,
1547                         struct usb_gadget_driver *driver)
1548 {
1549         struct ci13xxx *ci = container_of(gadget, struct ci13xxx, gadget);
1550         unsigned long flags;
1551
1552         spin_lock_irqsave(&ci->lock, flags);
1553
1554         if (!(ci->platdata->flags & CI13XXX_PULLUP_ON_VBUS) ||
1555                         ci->vbus_active) {
1556                 hw_device_state(ci, 0);
1557                 if (ci->platdata->notify_event)
1558                         ci->platdata->notify_event(ci,
1559                         CI13XXX_CONTROLLER_STOPPED_EVENT);
1560                 ci->driver = NULL;
1561                 spin_unlock_irqrestore(&ci->lock, flags);
1562                 _gadget_stop_activity(&ci->gadget);
1563                 spin_lock_irqsave(&ci->lock, flags);
1564                 pm_runtime_put(&ci->gadget.dev);
1565         }
1566
1567         spin_unlock_irqrestore(&ci->lock, flags);
1568
1569         return 0;
1570 }
1571
1572 /******************************************************************************
1573  * BUS block
1574  *****************************************************************************/
1575 /**
1576  * udc_irq: ci interrupt handler
1577  *
1578  * This function returns IRQ_HANDLED if the IRQ has been handled
1579  * It locks access to registers
1580  */
1581 static irqreturn_t udc_irq(struct ci13xxx *ci)
1582 {
1583         irqreturn_t retval;
1584         u32 intr;
1585
1586         if (ci == NULL)
1587                 return IRQ_HANDLED;
1588
1589         spin_lock(&ci->lock);
1590
1591         if (ci->platdata->flags & CI13XXX_REGS_SHARED) {
1592                 if (hw_read(ci, OP_USBMODE, USBMODE_CM) !=
1593                                 USBMODE_CM_DC) {
1594                         spin_unlock(&ci->lock);
1595                         return IRQ_NONE;
1596                 }
1597         }
1598         intr = hw_test_and_clear_intr_active(ci);
1599         dbg_interrupt(intr);
1600
1601         if (intr) {
1602                 /* order defines priority - do NOT change it */
1603                 if (USBi_URI & intr)
1604                         isr_reset_handler(ci);
1605
1606                 if (USBi_PCI & intr) {
1607                         ci->gadget.speed = hw_port_is_high_speed(ci) ?
1608                                 USB_SPEED_HIGH : USB_SPEED_FULL;
1609                         if (ci->suspended && ci->driver->resume) {
1610                                 spin_unlock(&ci->lock);
1611                                 ci->driver->resume(&ci->gadget);
1612                                 spin_lock(&ci->lock);
1613                                 ci->suspended = 0;
1614                         }
1615                 }
1616
1617                 if (USBi_UI  & intr)
1618                         isr_tr_complete_handler(ci);
1619
1620                 if (USBi_SLI & intr) {
1621                         if (ci->gadget.speed != USB_SPEED_UNKNOWN &&
1622                             ci->driver->suspend) {
1623                                 ci->suspended = 1;
1624                                 spin_unlock(&ci->lock);
1625                                 ci->driver->suspend(&ci->gadget);
1626                                 spin_lock(&ci->lock);
1627                         }
1628                 }
1629                 retval = IRQ_HANDLED;
1630         } else {
1631                 retval = IRQ_NONE;
1632         }
1633         spin_unlock(&ci->lock);
1634
1635         return retval;
1636 }
1637
1638 /**
1639  * udc_release: driver release function
1640  * @dev: device
1641  *
1642  * Currently does nothing
1643  */
1644 static void udc_release(struct device *dev)
1645 {
1646 }
1647
1648 /**
1649  * udc_start: initialize gadget role
1650  * @ci: chipidea controller
1651  */
1652 static int udc_start(struct ci13xxx *ci)
1653 {
1654         struct device *dev = ci->dev;
1655         int retval = 0;
1656
1657         spin_lock_init(&ci->lock);
1658
1659         ci->gadget.ops          = &usb_gadget_ops;
1660         ci->gadget.speed        = USB_SPEED_UNKNOWN;
1661         ci->gadget.max_speed    = USB_SPEED_HIGH;
1662         ci->gadget.is_otg       = 0;
1663         ci->gadget.name         = ci->platdata->name;
1664
1665         INIT_LIST_HEAD(&ci->gadget.ep_list);
1666
1667         dev_set_name(&ci->gadget.dev, "gadget");
1668         ci->gadget.dev.dma_mask = dev->dma_mask;
1669         ci->gadget.dev.coherent_dma_mask = dev->coherent_dma_mask;
1670         ci->gadget.dev.parent   = dev;
1671         ci->gadget.dev.release  = udc_release;
1672
1673         /* alloc resources */
1674         ci->qh_pool = dma_pool_create("ci13xxx_qh", dev,
1675                                        sizeof(struct ci13xxx_qh),
1676                                        64, CI13XXX_PAGE_SIZE);
1677         if (ci->qh_pool == NULL)
1678                 return -ENOMEM;
1679
1680         ci->td_pool = dma_pool_create("ci13xxx_td", dev,
1681                                        sizeof(struct ci13xxx_td),
1682                                        64, CI13XXX_PAGE_SIZE);
1683         if (ci->td_pool == NULL) {
1684                 retval = -ENOMEM;
1685                 goto free_qh_pool;
1686         }
1687
1688         retval = init_eps(ci);
1689         if (retval)
1690                 goto free_pools;
1691
1692         ci->gadget.ep0 = &ci->ep0in->ep;
1693
1694         if (ci->global_phy)
1695                 ci->transceiver = usb_get_phy(USB_PHY_TYPE_USB2);
1696
1697         if (ci->platdata->flags & CI13XXX_REQUIRE_TRANSCEIVER) {
1698                 if (ci->transceiver == NULL) {
1699                         retval = -ENODEV;
1700                         goto free_pools;
1701                 }
1702         }
1703
1704         if (!(ci->platdata->flags & CI13XXX_REGS_SHARED)) {
1705                 retval = hw_device_reset(ci, USBMODE_CM_DC);
1706                 if (retval)
1707                         goto put_transceiver;
1708         }
1709
1710         retval = device_register(&ci->gadget.dev);
1711         if (retval) {
1712                 put_device(&ci->gadget.dev);
1713                 goto put_transceiver;
1714         }
1715
1716         retval = dbg_create_files(&ci->gadget.dev);
1717         if (retval)
1718                 goto unreg_device;
1719
1720         if (!IS_ERR_OR_NULL(ci->transceiver)) {
1721                 retval = otg_set_peripheral(ci->transceiver->otg,
1722                                                 &ci->gadget);
1723                 if (retval)
1724                         goto remove_dbg;
1725         }
1726
1727         retval = usb_add_gadget_udc(dev, &ci->gadget);
1728         if (retval)
1729                 goto remove_trans;
1730
1731         pm_runtime_no_callbacks(&ci->gadget.dev);
1732         pm_runtime_enable(&ci->gadget.dev);
1733
1734         return retval;
1735
1736 remove_trans:
1737         if (!IS_ERR_OR_NULL(ci->transceiver)) {
1738                 otg_set_peripheral(ci->transceiver->otg, &ci->gadget);
1739                 if (ci->global_phy)
1740                         usb_put_phy(ci->transceiver);
1741         }
1742
1743         dev_err(dev, "error = %i\n", retval);
1744 remove_dbg:
1745         dbg_remove_files(&ci->gadget.dev);
1746 unreg_device:
1747         device_unregister(&ci->gadget.dev);
1748 put_transceiver:
1749         if (!IS_ERR_OR_NULL(ci->transceiver) && ci->global_phy)
1750                 usb_put_phy(ci->transceiver);
1751 free_pools:
1752         dma_pool_destroy(ci->td_pool);
1753 free_qh_pool:
1754         dma_pool_destroy(ci->qh_pool);
1755         return retval;
1756 }
1757
1758 /**
1759  * udc_remove: parent remove must call this to remove UDC
1760  *
1761  * No interrupts active, the IRQ has been released
1762  */
1763 static void udc_stop(struct ci13xxx *ci)
1764 {
1765         int i;
1766
1767         if (ci == NULL)
1768                 return;
1769
1770         usb_del_gadget_udc(&ci->gadget);
1771
1772         for (i = 0; i < ci->hw_ep_max; i++) {
1773                 struct ci13xxx_ep *mEp = &ci->ci13xxx_ep[i];
1774
1775                 dma_pool_free(ci->qh_pool, mEp->qh.ptr, mEp->qh.dma);
1776         }
1777
1778         dma_pool_destroy(ci->td_pool);
1779         dma_pool_destroy(ci->qh_pool);
1780
1781         if (!IS_ERR_OR_NULL(ci->transceiver)) {
1782                 otg_set_peripheral(ci->transceiver->otg, NULL);
1783                 if (ci->global_phy)
1784                         usb_put_phy(ci->transceiver);
1785         }
1786         dbg_remove_files(&ci->gadget.dev);
1787         device_unregister(&ci->gadget.dev);
1788         /* my kobject is dynamic, I swear! */
1789         memset(&ci->gadget, 0, sizeof(ci->gadget));
1790 }
1791
1792 /**
1793  * ci_hdrc_gadget_init - initialize device related bits
1794  * ci: the controller
1795  *
1796  * This function enables the gadget role, if the device is "device capable".
1797  */
1798 int ci_hdrc_gadget_init(struct ci13xxx *ci)
1799 {
1800         struct ci_role_driver *rdrv;
1801
1802         if (!hw_read(ci, CAP_DCCPARAMS, DCCPARAMS_DC))
1803                 return -ENXIO;
1804
1805         rdrv = devm_kzalloc(ci->dev, sizeof(struct ci_role_driver), GFP_KERNEL);
1806         if (!rdrv)
1807                 return -ENOMEM;
1808
1809         rdrv->start     = udc_start;
1810         rdrv->stop      = udc_stop;
1811         rdrv->irq       = udc_irq;
1812         rdrv->name      = "gadget";
1813         ci->roles[CI_ROLE_GADGET] = rdrv;
1814
1815         return 0;
1816 }