drivers/tty/serial/8250/8250_omap.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * 8250-core based driver for the OMAP internal UART
   4  *
   5  * based on omap-serial.c, Copyright (C) 2010 Texas Instruments.
   6  *
   7  * Copyright (C) 2014 Sebastian Andrzej Siewior
   8  *
   9  */
  10
  11 #include <linux/clk.h>
  12 #include <linux/device.h>
  13 #include <linux/io.h>
  14 #include <linux/module.h>
  15 #include <linux/serial_8250.h>
  16 #include <linux/serial_reg.h>
  17 #include <linux/tty_flip.h>
  18 #include <linux/platform_device.h>
  19 #include <linux/slab.h>
  20 #include <linux/of.h>
  21 #include <linux/of_device.h>
  22 #include <linux/of_gpio.h>
  23 #include <linux/of_irq.h>
  24 #include <linux/delay.h>
  25 #include <linux/pm_runtime.h>
  26 #include <linux/console.h>
  27 #include <linux/pm_qos.h>
  28 #include <linux/pm_wakeirq.h>
  29 #include <linux/dma-mapping.h>
  30
  31 #include "8250.h"
  32
  33 #define DEFAULT_CLK_SPEED       48000000
  34
  35 #define UART_ERRATA_i202_MDR1_ACCESS    (1 << 0)
  36 #define OMAP_UART_WER_HAS_TX_WAKEUP     (1 << 1)
  37 #define OMAP_DMA_TX_KICK                (1 << 2)
  38 /*
  39  * See Advisory 21 in AM437x errata SPRZ408B, updated April 2015.
  40  * The same errata is applicable to AM335x and DRA7x processors too.
  41  */
  42 #define UART_ERRATA_CLOCK_DISABLE       (1 << 3)
  43 #define UART_HAS_EFR2                   BIT(4)
  44
  45 #define OMAP_UART_FCR_RX_TRIG           6
  46 #define OMAP_UART_FCR_TX_TRIG           4
  47
  48 /* SCR register bitmasks */
  49 #define OMAP_UART_SCR_RX_TRIG_GRANU1_MASK       (1 << 7)
  50 #define OMAP_UART_SCR_TX_TRIG_GRANU1_MASK       (1 << 6)
  51 #define OMAP_UART_SCR_TX_EMPTY                  (1 << 3)
  52 #define OMAP_UART_SCR_DMAMODE_MASK              (3 << 1)
  53 #define OMAP_UART_SCR_DMAMODE_1                 (1 << 1)
  54 #define OMAP_UART_SCR_DMAMODE_CTL               (1 << 0)
  55
  56 /* MVR register bitmasks */
  57 #define OMAP_UART_MVR_SCHEME_SHIFT      30
  58 #define OMAP_UART_LEGACY_MVR_MAJ_MASK   0xf0
  59 #define OMAP_UART_LEGACY_MVR_MAJ_SHIFT  4
  60 #define OMAP_UART_LEGACY_MVR_MIN_MASK   0x0f
  61 #define OMAP_UART_MVR_MAJ_MASK          0x700
  62 #define OMAP_UART_MVR_MAJ_SHIFT         8
  63 #define OMAP_UART_MVR_MIN_MASK          0x3f
  64
  65 /* SYSC register bitmasks */
  66 #define OMAP_UART_SYSC_SOFTRESET        (1 << 1)
  67
  68 /* SYSS register bitmasks */
  69 #define OMAP_UART_SYSS_RESETDONE        (1 << 0)
  70
  71 #define UART_TI752_TLR_TX       0
  72 #define UART_TI752_TLR_RX       4
  73
  74 #define TRIGGER_TLR_MASK(x)     ((x & 0x3c) >> 2)
  75 #define TRIGGER_FCR_MASK(x)     (x & 3)
  76
  77 /* Enable XON/XOFF flow control on output */
  78 #define OMAP_UART_SW_TX         0x08
  79 /* Enable XON/XOFF flow control on input */
  80 #define OMAP_UART_SW_RX         0x02
  81
  82 #define OMAP_UART_WER_MOD_WKUP  0x7f
  83 #define OMAP_UART_TX_WAKEUP_EN  (1 << 7)
  84
  85 #define TX_TRIGGER      1
  86 #define RX_TRIGGER      48
  87
  88 #define OMAP_UART_TCR_RESTORE(x)        ((x / 4) << 4)
  89 #define OMAP_UART_TCR_HALT(x)           ((x / 4) << 0)
  90
  91 #define UART_BUILD_REVISION(x, y)       (((x) << 8) | (y))
  92
  93 #define OMAP_UART_REV_46 0x0406
  94 #define OMAP_UART_REV_52 0x0502
  95 #define OMAP_UART_REV_63 0x0603
  96
  97 /* Enhanced features register 2 */
  98 #define UART_OMAP_EFR2                  0x23
  99 #define UART_OMAP_EFR2_TIMEOUT_BEHAVE   BIT(6)
 100
 101 struct omap8250_priv {
 102         int line;
 103         u8 habit;
 104         u8 mdr1;
 105         u8 efr;
 106         u8 scr;
 107         u8 wer;
 108         u8 xon;
 109         u8 xoff;
 110         u8 delayed_restore;
 111         u16 quot;
 112
 113         u8 tx_trigger;
 114         u8 rx_trigger;
 115         bool is_suspending;
 116         int wakeirq;
 117         int wakeups_enabled;
 118         u32 latency;
 119         u32 calc_latency;
 120         struct pm_qos_request pm_qos_request;
 121         struct work_struct qos_work;
 122         struct uart_8250_dma omap8250_dma;
 123         spinlock_t rx_dma_lock;
 124         bool rx_dma_broken;
 125         bool throttled;
 126 };
 127
 128 struct omap8250_dma_params {
 129         u32 rx_size;
 130         u8 rx_trigger;
 131         u8 tx_trigger;
 132 };
 133
 134 struct omap8250_platdata {
 135         struct omap8250_dma_params *dma_params;
 136         u8 habit;
 137 };
 138
 139 #ifdef CONFIG_SERIAL_8250_DMA
 140 static void omap_8250_rx_dma_flush(struct uart_8250_port *p);
 141 #else
 142 static inline void omap_8250_rx_dma_flush(struct uart_8250_port *p) { }
 143 #endif
 144
 145 static u32 uart_read(struct uart_8250_port *up, u32 reg)
 146 {
 147         return readl(up->port.membase + (reg << up->port.regshift));
 148 }
 149
 150 static void omap8250_set_mctrl(struct uart_port *port, unsigned int mctrl)
 151 {
 152         struct uart_8250_port *up = up_to_u8250p(port);
 153         struct omap8250_priv *priv = up->port.private_data;
 154         u8 lcr;
 155
 156         serial8250_do_set_mctrl(port, mctrl);
 157
 158         if (!mctrl_gpio_to_gpiod(up->gpios, UART_GPIO_RTS)) {
 159                 /*
 160                  * Turn off autoRTS if RTS is lowered and restore autoRTS
 161                  * setting if RTS is raised
 162                  */
 163                 lcr = serial_in(up, UART_LCR);
 164                 serial_out(up, UART_LCR, UART_LCR_CONF_MODE_B);
 165                 if ((mctrl & TIOCM_RTS) && (port->status & UPSTAT_AUTORTS))
 166                         priv->efr |= UART_EFR_RTS;
 167                 else
 168                         priv->efr &= ~UART_EFR_RTS;
 169                 serial_out(up, UART_EFR, priv->efr);
 170                 serial_out(up, UART_LCR, lcr);
 171         }
 172 }
 173
 174 /*
 175  * Work Around for Errata i202 (2430, 3430, 3630, 4430 and 4460)
 176  * The access to uart register after MDR1 Access
 177  * causes UART to corrupt data.
 178  *
 179  * Need a delay =
 180  * 5 L4 clock cycles + 5 UART functional clock cycle (@48MHz = ~0.2uS)
 181  * give 10 times as much
 182  */
 183 static void omap_8250_mdr1_errataset(struct uart_8250_port *up,
 184                                      struct omap8250_priv *priv)
 185 {
 186         u8 timeout = 255;
 187         u8 old_mdr1;
 188
 189         old_mdr1 = serial_in(up, UART_OMAP_MDR1);
 190         if (old_mdr1 == priv->mdr1)
 191                 return;
 192
 193         serial_out(up, UART_OMAP_MDR1, priv->mdr1);
 194         udelay(2);
 195         serial_out(up, UART_FCR, up->fcr | UART_FCR_CLEAR_XMIT |
 196                         UART_FCR_CLEAR_RCVR);
 197         /*
 198          * Wait for FIFO to empty: when empty, RX_FIFO_E bit is 0 and
 199          * TX_FIFO_E bit is 1.
 200          */
 201         while (UART_LSR_THRE != (serial_in(up, UART_LSR) &
 202                                 (UART_LSR_THRE | UART_LSR_DR))) {
 203                 timeout--;
 204                 if (!timeout) {
 205                         /* Should *never* happen. we warn and carry on */
 206                         dev_crit(up->port.dev, "Errata i202: timedout %x\n",
 207                                  serial_in(up, UART_LSR));
 208                         break;
 209                 }
 210                 udelay(1);
 211         }
 212 }
 213
 214 static void omap_8250_get_divisor(struct uart_port *port, unsigned int baud,
 215                                   struct omap8250_priv *priv)
 216 {
 217         unsigned int uartclk = port->uartclk;
 218         unsigned int div_13, div_16;
 219         unsigned int abs_d13, abs_d16;
 220
 221         /*
 222          * Old custom speed handling.
 223          */
 224         if (baud == 38400 && (port->flags & UPF_SPD_MASK) == UPF_SPD_CUST) {
 225                 priv->quot = port->custom_divisor & UART_DIV_MAX;
 226                 /*
 227                  * I assume that nobody is using this. But hey, if somebody
 228                  * would like to specify the divisor _and_ the mode then the
 229                  * driver is ready and waiting for it.
 230                  */
 231                 if (port->custom_divisor & (1 << 16))
 232                         priv->mdr1 = UART_OMAP_MDR1_13X_MODE;
 233                 else
 234                         priv->mdr1 = UART_OMAP_MDR1_16X_MODE;
 235                 return;
 236         }
 237         div_13 = DIV_ROUND_CLOSEST(uartclk, 13 * baud);
 238         div_16 = DIV_ROUND_CLOSEST(uartclk, 16 * baud);
 239
 240         if (!div_13)
 241                 div_13 = 1;
 242         if (!div_16)
 243                 div_16 = 1;
 244
 245         abs_d13 = abs(baud - uartclk / 13 / div_13);
 246         abs_d16 = abs(baud - uartclk / 16 / div_16);
 247
 248         if (abs_d13 >= abs_d16) {
 249                 priv->mdr1 = UART_OMAP_MDR1_16X_MODE;
 250                 priv->quot = div_16;
 251         } else {
 252                 priv->mdr1 = UART_OMAP_MDR1_13X_MODE;
 253                 priv->quot = div_13;
 254         }
 255 }
 256
 257 static void omap8250_update_scr(struct uart_8250_port *up,
 258                                 struct omap8250_priv *priv)
 259 {
 260         u8 old_scr;
 261
 262         old_scr = serial_in(up, UART_OMAP_SCR);
 263         if (old_scr == priv->scr)
 264                 return;
 265
 266         /*
 267          * The manual recommends not to enable the DMA mode selector in the SCR
 268          * (instead of the FCR) register _and_ selecting the DMA mode as one
 269          * register write because this may lead to malfunction.
 270          */
 271         if (priv->scr & OMAP_UART_SCR_DMAMODE_MASK)
 272                 serial_out(up, UART_OMAP_SCR,
 273                            priv->scr & ~OMAP_UART_SCR_DMAMODE_MASK);
 274         serial_out(up, UART_OMAP_SCR, priv->scr);
 275 }
 276
 277 static void omap8250_update_mdr1(struct uart_8250_port *up,
 278                                  struct omap8250_priv *priv)
 279 {
 280         if (priv->habit & UART_ERRATA_i202_MDR1_ACCESS)
 281                 omap_8250_mdr1_errataset(up, priv);
 282         else
 283                 serial_out(up, UART_OMAP_MDR1, priv->mdr1);
 284 }
 285
 286 static void omap8250_restore_regs(struct uart_8250_port *up)
 287 {
 288         struct omap8250_priv *priv = up->port.private_data;
 289         struct uart_8250_dma    *dma = up->dma;
 290
 291         if (dma && dma->tx_running) {
 292                 /*
 293                  * TCSANOW requests the change to occur immediately however if
 294                  * we have a TX-DMA operation in progress then it has been
 295                  * observed that it might stall and never complete. Therefore we
 296                  * delay DMA completes to prevent this hang from happen.
 297                  */
 298                 priv->delayed_restore = 1;
 299                 return;
 300         }
 301
 302         serial_out(up, UART_LCR, UART_LCR_CONF_MODE_B);
 303         serial_out(up, UART_EFR, UART_EFR_ECB);
 304
 305         serial_out(up, UART_LCR, UART_LCR_CONF_MODE_A);
 306         serial8250_out_MCR(up, UART_MCR_TCRTLR);
 307         serial_out(up, UART_FCR, up->fcr);
 308
 309         omap8250_update_scr(up, priv);
 310
 311         serial_out(up, UART_LCR, UART_LCR_CONF_MODE_B);
 312
 313         serial_out(up, UART_TI752_TCR, OMAP_UART_TCR_RESTORE(16) |
 314                         OMAP_UART_TCR_HALT(52));
 315         serial_out(up, UART_TI752_TLR,
 316                    TRIGGER_TLR_MASK(priv->tx_trigger) << UART_TI752_TLR_TX |
 317                    TRIGGER_TLR_MASK(priv->rx_trigger) << UART_TI752_TLR_RX);
 318
 319         serial_out(up, UART_LCR, 0);
 320
 321         /* drop TCR + TLR access, we setup XON/XOFF later */
 322         serial8250_out_MCR(up, up->mcr);
 323         serial_out(up, UART_IER, up->ier);
 324
 325         serial_out(up, UART_LCR, UART_LCR_CONF_MODE_B);
 326         serial_dl_write(up, priv->quot);
 327
 328         serial_out(up, UART_EFR, priv->efr);
 329
 330         /* Configure flow control */
 331         serial_out(up, UART_LCR, UART_LCR_CONF_MODE_B);
 332         serial_out(up, UART_XON1, priv->xon);
 333         serial_out(up, UART_XOFF1, priv->xoff);
 334
 335         serial_out(up, UART_LCR, up->lcr);
 336
 337         omap8250_update_mdr1(up, priv);
 338
 339         up->port.ops->set_mctrl(&up->port, up->port.mctrl);
 340 }
 341
 342 /*
 343  * OMAP can use "CLK / (16 or 13) / div" for baud rate. And then we have have
 344  * some differences in how we want to handle flow control.
 345  */
 346 static void omap_8250_set_termios(struct uart_port *port,
 347                                   struct ktermios *termios,
 348                                   struct ktermios *old)
 349 {
 350         struct uart_8250_port *up = up_to_u8250p(port);
 351         struct omap8250_priv *priv = up->port.private_data;
 352         unsigned char cval = 0;
 353         unsigned int baud;
 354
 355         switch (termios->c_cflag & CSIZE) {
 356         case CS5:
 357                 cval = UART_LCR_WLEN5;
 358                 break;
 359         case CS6:
 360                 cval = UART_LCR_WLEN6;
 361                 break;
 362         case CS7:
 363                 cval = UART_LCR_WLEN7;
 364                 break;
 365         default:
 366         case CS8:
 367                 cval = UART_LCR_WLEN8;
 368                 break;
 369         }
 370
 371         if (termios->c_cflag & CSTOPB)
 372                 cval |= UART_LCR_STOP;
 373         if (termios->c_cflag & PARENB)
 374                 cval |= UART_LCR_PARITY;
 375         if (!(termios->c_cflag & PARODD))
 376                 cval |= UART_LCR_EPAR;
 377         if (termios->c_cflag & CMSPAR)
 378                 cval |= UART_LCR_SPAR;
 379
 380         /*
 381          * Ask the core to calculate the divisor for us.
 382          */
 383         baud = uart_get_baud_rate(port, termios, old,
 384                                   port->uartclk / 16 / UART_DIV_MAX,
 385                                   port->uartclk / 13);
 386         omap_8250_get_divisor(port, baud, priv);
 387
 388         /*
 389          * Ok, we're now changing the port state. Do it with
 390          * interrupts disabled.
 391          */
 392         pm_runtime_get_sync(port->dev);
 393         spin_lock_irq(&port->lock);
 394
 395         /*
 396          * Update the per-port timeout.
 397          */
 398         uart_update_timeout(port, termios->c_cflag, baud);
 399
 400         up->port.read_status_mask = UART_LSR_OE | UART_LSR_THRE | UART_LSR_DR;
 401         if (termios->c_iflag & INPCK)
 402                 up->port.read_status_mask |= UART_LSR_FE | UART_LSR_PE;
 403         if (termios->c_iflag & (IGNBRK | PARMRK))
 404                 up->port.read_status_mask |= UART_LSR_BI;
 405
 406         /*
 407          * Characters to ignore
 408          */
 409         up->port.ignore_status_mask = 0;
 410         if (termios->c_iflag & IGNPAR)
 411                 up->port.ignore_status_mask |= UART_LSR_PE | UART_LSR_FE;
 412         if (termios->c_iflag & IGNBRK) {
 413                 up->port.ignore_status_mask |= UART_LSR_BI;
 414                 /*
 415                  * If we're ignoring parity and break indicators,
 416                  * ignore overruns too (for real raw support).
 417                  */
 418                 if (termios->c_iflag & IGNPAR)
 419                         up->port.ignore_status_mask |= UART_LSR_OE;
 420         }
 421
 422         /*
 423          * ignore all characters if CREAD is not set
 424          */
 425         if ((termios->c_cflag & CREAD) == 0)
 426                 up->port.ignore_status_mask |= UART_LSR_DR;
 427
 428         /*
 429          * Modem status interrupts
 430          */
 431         up->ier &= ~UART_IER_MSI;
 432         if (UART_ENABLE_MS(&up->port, termios->c_cflag))
 433                 up->ier |= UART_IER_MSI;
 434
 435         up->lcr = cval;
 436         /* Up to here it was mostly serial8250_do_set_termios() */
 437
 438         /*
 439          * We enable TRIG_GRANU for RX and TX and additionally we set
 440          * SCR_TX_EMPTY bit. The result is the following:
 441          * - RX_TRIGGER amount of bytes in the FIFO will cause an interrupt.
 442          * - less than RX_TRIGGER number of bytes will also cause an interrupt
 443          *   once the UART decides that there no new bytes arriving.
 444          * - Once THRE is enabled, the interrupt will be fired once the FIFO is
 445          *   empty - the trigger level is ignored here.
 446          *
 447          * Once DMA is enabled:
 448          * - UART will assert the TX DMA line once there is room for TX_TRIGGER
 449          *   bytes in the TX FIFO. On each assert the DMA engine will move
 450          *   TX_TRIGGER bytes into the FIFO.
 451          * - UART will assert the RX DMA line once there are RX_TRIGGER bytes in
 452          *   the FIFO and move RX_TRIGGER bytes.
 453          * This is because threshold and trigger values are the same.
 454          */
 455         up->fcr = UART_FCR_ENABLE_FIFO;
 456         up->fcr |= TRIGGER_FCR_MASK(priv->tx_trigger) << OMAP_UART_FCR_TX_TRIG;
 457         up->fcr |= TRIGGER_FCR_MASK(priv->rx_trigger) << OMAP_UART_FCR_RX_TRIG;
 458
 459         priv->scr = OMAP_UART_SCR_RX_TRIG_GRANU1_MASK | OMAP_UART_SCR_TX_EMPTY |
 460                 OMAP_UART_SCR_TX_TRIG_GRANU1_MASK;
 461
 462         if (up->dma)
 463                 priv->scr |= OMAP_UART_SCR_DMAMODE_1 |
 464                         OMAP_UART_SCR_DMAMODE_CTL;
 465
 466         priv->xon = termios->c_cc[VSTART];
 467         priv->xoff = termios->c_cc[VSTOP];
 468
 469         priv->efr = 0;
 470         up->port.status &= ~(UPSTAT_AUTOCTS | UPSTAT_AUTORTS | UPSTAT_AUTOXOFF);
 471
 472         if (termios->c_cflag & CRTSCTS && up->port.flags & UPF_HARD_FLOW &&
 473             !mctrl_gpio_to_gpiod(up->gpios, UART_GPIO_RTS) &&
 474             !mctrl_gpio_to_gpiod(up->gpios, UART_GPIO_CTS)) {
 475                 /* Enable AUTOCTS (autoRTS is enabled when RTS is raised) */
 476                 up->port.status |= UPSTAT_AUTOCTS | UPSTAT_AUTORTS;
 477                 priv->efr |= UART_EFR_CTS;
 478         } else  if (up->port.flags & UPF_SOFT_FLOW) {
 479                 /*
 480                  * OMAP rx s/w flow control is borked; the transmitter remains
 481                  * stuck off even if rx flow control is subsequently disabled
 482                  */
 483
 484                 /*
 485                  * IXOFF Flag:
 486                  * Enable XON/XOFF flow control on output.
 487                  * Transmit XON1, XOFF1
 488                  */
 489                 if (termios->c_iflag & IXOFF) {
 490                         up->port.status |= UPSTAT_AUTOXOFF;
 491                         priv->efr |= OMAP_UART_SW_TX;
 492                 }
 493         }
 494         omap8250_restore_regs(up);
 495
 496         spin_unlock_irq(&up->port.lock);
 497         pm_runtime_mark_last_busy(port->dev);
 498         pm_runtime_put_autosuspend(port->dev);
 499
 500         /* calculate wakeup latency constraint */
 501         priv->calc_latency = USEC_PER_SEC * 64 * 8 / baud;
 502         priv->latency = priv->calc_latency;
 503
 504         schedule_work(&priv->qos_work);
 505
 506         /* Don't rewrite B0 */
 507         if (tty_termios_baud_rate(termios))
 508                 tty_termios_encode_baud_rate(termios, baud, baud);
 509 }
 510
 511 /* same as 8250 except that we may have extra flow bits set in EFR */
 512 static void omap_8250_pm(struct uart_port *port, unsigned int state,
 513                          unsigned int oldstate)
 514 {
 515         struct uart_8250_port *up = up_to_u8250p(port);
 516         u8 efr;
 517
 518         pm_runtime_get_sync(port->dev);
 519         serial_out(up, UART_LCR, UART_LCR_CONF_MODE_B);
 520         efr = serial_in(up, UART_EFR);
 521         serial_out(up, UART_EFR, efr | UART_EFR_ECB);
 522         serial_out(up, UART_LCR, 0);
 523
 524         serial_out(up, UART_IER, (state != 0) ? UART_IERX_SLEEP : 0);
 525         serial_out(up, UART_LCR, UART_LCR_CONF_MODE_B);
 526         serial_out(up, UART_EFR, efr);
 527         serial_out(up, UART_LCR, 0);
 528
 529         pm_runtime_mark_last_busy(port->dev);
 530         pm_runtime_put_autosuspend(port->dev);
 531 }
 532
 533 static void omap_serial_fill_features_erratas(struct uart_8250_port *up,
 534                                               struct omap8250_priv *priv)
 535 {
 536         u32 mvr, scheme;
 537         u16 revision, major, minor;
 538
 539         mvr = uart_read(up, UART_OMAP_MVER);
 540
 541         /* Check revision register scheme */
 542         scheme = mvr >> OMAP_UART_MVR_SCHEME_SHIFT;
 543
 544         switch (scheme) {
 545         case 0: /* Legacy Scheme: OMAP2/3 */
 546                 /* MINOR_REV[0:4], MAJOR_REV[4:7] */
 547                 major = (mvr & OMAP_UART_LEGACY_MVR_MAJ_MASK) >>
 548                         OMAP_UART_LEGACY_MVR_MAJ_SHIFT;
 549                 minor = (mvr & OMAP_UART_LEGACY_MVR_MIN_MASK);
 550                 break;
 551         case 1:
 552                 /* New Scheme: OMAP4+ */
 553                 /* MINOR_REV[0:5], MAJOR_REV[8:10] */
 554                 major = (mvr & OMAP_UART_MVR_MAJ_MASK) >>
 555                         OMAP_UART_MVR_MAJ_SHIFT;
 556                 minor = (mvr & OMAP_UART_MVR_MIN_MASK);
 557                 break;
 558         default:
 559                 dev_warn(up->port.dev,
 560                          "Unknown revision, defaulting to highest\n");
 561                 /* highest possible revision */
 562                 major = 0xff;
 563                 minor = 0xff;
 564         }
 565         /* normalize revision for the driver */
 566         revision = UART_BUILD_REVISION(major, minor);
 567
 568         switch (revision) {
 569         case OMAP_UART_REV_46:
 570                 priv->habit |= UART_ERRATA_i202_MDR1_ACCESS;
 571                 break;
 572         case OMAP_UART_REV_52:
 573                 priv->habit |= UART_ERRATA_i202_MDR1_ACCESS |
 574                                 OMAP_UART_WER_HAS_TX_WAKEUP;
 575                 break;
 576         case OMAP_UART_REV_63:
 577                 priv->habit |= UART_ERRATA_i202_MDR1_ACCESS |
 578                         OMAP_UART_WER_HAS_TX_WAKEUP;
 579                 break;
 580         default:
 581                 break;
 582         }
 583 }
 584
 585 static void omap8250_uart_qos_work(struct work_struct *work)
 586 {
 587         struct omap8250_priv *priv;
 588
 589         priv = container_of(work, struct omap8250_priv, qos_work);
 590         cpu_latency_qos_update_request(&priv->pm_qos_request, priv->latency);
 591 }
 592
 593 #ifdef CONFIG_SERIAL_8250_DMA
 594 static int omap_8250_dma_handle_irq(struct uart_port *port);
 595 #endif
 596
 597 static irqreturn_t omap8250_irq(int irq, void *dev_id)
 598 {
 599         struct uart_port *port = dev_id;
 600         struct uart_8250_port *up = up_to_u8250p(port);
 601         unsigned int iir;
 602         int ret;
 603
 604 #ifdef CONFIG_SERIAL_8250_DMA
 605         if (up->dma) {
 606                 ret = omap_8250_dma_handle_irq(port);
 607                 return IRQ_RETVAL(ret);
 608         }
 609 #endif
 610
 611         serial8250_rpm_get(up);
 612         iir = serial_port_in(port, UART_IIR);
 613         ret = serial8250_handle_irq(port, iir);
 614         serial8250_rpm_put(up);
 615
 616         return IRQ_RETVAL(ret);
 617 }
 618
 619 static int omap_8250_startup(struct uart_port *port)
 620 {
 621         struct uart_8250_port *up = up_to_u8250p(port);
 622         struct omap8250_priv *priv = port->private_data;
 623         int ret;
 624
 625         if (priv->wakeirq) {
 626                 ret = dev_pm_set_dedicated_wake_irq(port->dev, priv->wakeirq);
 627                 if (ret)
 628                         return ret;
 629         }
 630
 631         pm_runtime_get_sync(port->dev);
 632
 633         up->mcr = 0;
 634         serial_out(up, UART_FCR, UART_FCR_CLEAR_RCVR | UART_FCR_CLEAR_XMIT);
 635
 636         serial_out(up, UART_LCR, UART_LCR_WLEN8);
 637
 638         up->lsr_saved_flags = 0;
 639         up->msr_saved_flags = 0;
 640
 641         /* Disable DMA for console UART */
 642         if (uart_console(port))
 643                 up->dma = NULL;
 644
 645         if (up->dma) {
 646                 ret = serial8250_request_dma(up);
 647                 if (ret) {
 648                         dev_warn_ratelimited(port->dev,
 649                                              "failed to request DMA\n");
 650                         up->dma = NULL;
 651                 }
 652         }
 653
 654         ret = request_irq(port->irq, omap8250_irq, IRQF_SHARED,
 655                           dev_name(port->dev), port);
 656         if (ret < 0)
 657                 goto err;
 658
 659         up->ier = UART_IER_RLSI | UART_IER_RDI;
 660         serial_out(up, UART_IER, up->ier);
 661
 662 #ifdef CONFIG_PM
 663         up->capabilities |= UART_CAP_RPM;
 664 #endif
 665
 666         /* Enable module level wake up */
 667         priv->wer = OMAP_UART_WER_MOD_WKUP;
 668         if (priv->habit & OMAP_UART_WER_HAS_TX_WAKEUP)
 669                 priv->wer |= OMAP_UART_TX_WAKEUP_EN;
 670         serial_out(up, UART_OMAP_WER, priv->wer);
 671
 672         if (up->dma && !(priv->habit & UART_HAS_EFR2))
 673                 up->dma->rx_dma(up);
 674
 675         pm_runtime_mark_last_busy(port->dev);
 676         pm_runtime_put_autosuspend(port->dev);
 677         return 0;
 678 err:
 679         pm_runtime_mark_last_busy(port->dev);
 680         pm_runtime_put_autosuspend(port->dev);
 681         dev_pm_clear_wake_irq(port->dev);
 682         return ret;
 683 }
 684
 685 static void omap_8250_shutdown(struct uart_port *port)
 686 {
 687         struct uart_8250_port *up = up_to_u8250p(port);
 688         struct omap8250_priv *priv = port->private_data;
 689
 690         flush_work(&priv->qos_work);
 691         if (up->dma)
 692                 omap_8250_rx_dma_flush(up);
 693
 694         pm_runtime_get_sync(port->dev);
 695
 696         serial_out(up, UART_OMAP_WER, 0);
 697         if (priv->habit & UART_HAS_EFR2)
 698                 serial_out(up, UART_OMAP_EFR2, 0x0);
 699
 700         up->ier = 0;
 701         serial_out(up, UART_IER, 0);
 702
 703         if (up->dma)
 704                 serial8250_release_dma(up);
 705
 706         /*
 707          * Disable break condition and FIFOs
 708          */
 709         if (up->lcr & UART_LCR_SBC)
 710                 serial_out(up, UART_LCR, up->lcr & ~UART_LCR_SBC);
 711         serial_out(up, UART_FCR, UART_FCR_CLEAR_RCVR | UART_FCR_CLEAR_XMIT);
 712
 713         pm_runtime_mark_last_busy(port->dev);
 714         pm_runtime_put_autosuspend(port->dev);
 715         free_irq(port->irq, port);
 716         dev_pm_clear_wake_irq(port->dev);
 717 }
 718
 719 static void omap_8250_throttle(struct uart_port *port)
 720 {
 721         struct omap8250_priv *priv = port->private_data;
 722         unsigned long flags;
 723
 724         pm_runtime_get_sync(port->dev);
 725
 726         spin_lock_irqsave(&port->lock, flags);
 727         port->ops->stop_rx(port);
 728         priv->throttled = true;
 729         spin_unlock_irqrestore(&port->lock, flags);
 730
 731         pm_runtime_mark_last_busy(port->dev);
 732         pm_runtime_put_autosuspend(port->dev);
 733 }
 734
 735 static void omap_8250_unthrottle(struct uart_port *port)
 736 {
 737         struct omap8250_priv *priv = port->private_data;
 738         struct uart_8250_port *up = up_to_u8250p(port);
 739         unsigned long flags;
 740
 741         pm_runtime_get_sync(port->dev);
 742
 743         spin_lock_irqsave(&port->lock, flags);
 744         priv->throttled = false;
 745         if (up->dma)
 746                 up->dma->rx_dma(up);
 747         up->ier |= UART_IER_RLSI | UART_IER_RDI;
 748         port->read_status_mask |= UART_LSR_DR;
 749         serial_out(up, UART_IER, up->ier);
 750         spin_unlock_irqrestore(&port->lock, flags);
 751
 752         pm_runtime_mark_last_busy(port->dev);
 753         pm_runtime_put_autosuspend(port->dev);
 754 }
 755
 756 #ifdef CONFIG_SERIAL_8250_DMA
 757 static int omap_8250_rx_dma(struct uart_8250_port *p);
 758
 759 /* Must be called while priv->rx_dma_lock is held */
 760 static void __dma_rx_do_complete(struct uart_8250_port *p)
 761 {
 762         struct uart_8250_dma    *dma = p->dma;
 763         struct tty_port         *tty_port = &p->port.state->port;
 764         struct dma_chan         *rxchan = dma->rxchan;
 765         dma_cookie_t            cookie;
 766         struct dma_tx_state     state;
 767         int                     count;
 768         int                     ret;
 769
 770         if (!dma->rx_running)
 771                 goto out;
 772
 773         cookie = dma->rx_cookie;
 774         dma->rx_running = 0;
 775         dmaengine_tx_status(rxchan, cookie, &state);
 776
 777         count = dma->rx_size - state.residue + state.in_flight_bytes;
 778         if (count < dma->rx_size) {
 779                 dmaengine_terminate_async(rxchan);
 780
 781                 /*
 782                  * Poll for teardown to complete which guarantees in
 783                  * flight data is drained.
 784                  */
 785                 if (state.in_flight_bytes) {
 786                         int poll_count = 25;
 787
 788                         while (dmaengine_tx_status(rxchan, cookie, NULL) &&
 789                                poll_count--)
 790                                 cpu_relax();
 791
 792                         if (!poll_count)
 793                                 dev_err(p->port.dev, "teardown incomplete\n");
 794                 }
 795         }
 796         if (!count)
 797                 goto out;
 798         ret = tty_insert_flip_string(tty_port, dma->rx_buf, count);
 799
 800         p->port.icount.rx += ret;
 801         p->port.icount.buf_overrun += count - ret;
 802 out:
 803
 804         tty_flip_buffer_push(tty_port);
 805 }
 806
 807 static void __dma_rx_complete(void *param)
 808 {
 809         struct uart_8250_port *p = param;
 810         struct omap8250_priv *priv = p->port.private_data;
 811         struct uart_8250_dma *dma = p->dma;
 812         struct dma_tx_state     state;
 813         unsigned long flags;
 814
 815         spin_lock_irqsave(&p->port.lock, flags);
 816
 817         /*
 818          * If the tx status is not DMA_COMPLETE, then this is a delayed
 819          * completion callback. A previous RX timeout flush would have
 820          * already pushed the data, so exit.
 821          */
 822         if (dmaengine_tx_status(dma->rxchan, dma->rx_cookie, &state) !=
 823                         DMA_COMPLETE) {
 824                 spin_unlock_irqrestore(&p->port.lock, flags);
 825                 return;
 826         }
 827         __dma_rx_do_complete(p);
 828         if (!priv->throttled) {
 829                 p->ier |= UART_IER_RLSI | UART_IER_RDI;
 830                 serial_out(p, UART_IER, p->ier);
 831                 if (!(priv->habit & UART_HAS_EFR2))
 832                         omap_8250_rx_dma(p);
 833         }
 834
 835         spin_unlock_irqrestore(&p->port.lock, flags);
 836 }
 837
 838 static void omap_8250_rx_dma_flush(struct uart_8250_port *p)
 839 {
 840         struct omap8250_priv    *priv = p->port.private_data;
 841         struct uart_8250_dma    *dma = p->dma;
 842         struct dma_tx_state     state;
 843         unsigned long           flags;
 844         int ret;
 845
 846         spin_lock_irqsave(&priv->rx_dma_lock, flags);
 847
 848         if (!dma->rx_running) {
 849                 spin_unlock_irqrestore(&priv->rx_dma_lock, flags);
 850                 return;
 851         }
 852
 853         ret = dmaengine_tx_status(dma->rxchan, dma->rx_cookie, &state);
 854         if (ret == DMA_IN_PROGRESS) {
 855                 ret = dmaengine_pause(dma->rxchan);
 856                 if (WARN_ON_ONCE(ret))
 857                         priv->rx_dma_broken = true;
 858         }
 859         __dma_rx_do_complete(p);
 860         spin_unlock_irqrestore(&priv->rx_dma_lock, flags);
 861 }
 862
 863 static int omap_8250_rx_dma(struct uart_8250_port *p)
 864 {
 865         struct omap8250_priv            *priv = p->port.private_data;
 866         struct uart_8250_dma            *dma = p->dma;
 867         int                             err = 0;
 868         struct dma_async_tx_descriptor  *desc;
 869         unsigned long                   flags;
 870
 871         if (priv->rx_dma_broken)
 872                 return -EINVAL;
 873
 874         spin_lock_irqsave(&priv->rx_dma_lock, flags);
 875
 876         if (dma->rx_running) {
 877                 enum dma_status state;
 878
 879                 state = dmaengine_tx_status(dma->rxchan, dma->rx_cookie, NULL);
 880                 if (state == DMA_COMPLETE) {
 881                         /*
 882                          * Disable RX interrupts to allow RX DMA completion
 883                          * callback to run.
 884                          */
 885                         p->ier &= ~(UART_IER_RLSI | UART_IER_RDI);
 886                         serial_out(p, UART_IER, p->ier);
 887                 }
 888                 goto out;
 889         }
 890
 891         desc = dmaengine_prep_slave_single(dma->rxchan, dma->rx_addr,
 892                                            dma->rx_size, DMA_DEV_TO_MEM,
 893                                            DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 894         if (!desc) {
 895                 err = -EBUSY;
 896                 goto out;
 897         }
 898
 899         dma->rx_running = 1;
 900         desc->callback = __dma_rx_complete;
 901         desc->callback_param = p;
 902
 903         dma->rx_cookie = dmaengine_submit(desc);
 904
 905         dma_async_issue_pending(dma->rxchan);
 906 out:
 907         spin_unlock_irqrestore(&priv->rx_dma_lock, flags);
 908         return err;
 909 }
 910
 911 static int omap_8250_tx_dma(struct uart_8250_port *p);
 912
 913 static void omap_8250_dma_tx_complete(void *param)
 914 {
 915         struct uart_8250_port   *p = param;
 916         struct uart_8250_dma    *dma = p->dma;
 917         struct circ_buf         *xmit = &p->port.state->xmit;
 918         unsigned long           flags;
 919         bool                    en_thri = false;
 920         struct omap8250_priv    *priv = p->port.private_data;
 921
 922         dma_sync_single_for_cpu(dma->txchan->device->dev, dma->tx_addr,
 923                                 UART_XMIT_SIZE, DMA_TO_DEVICE);
 924
 925         spin_lock_irqsave(&p->port.lock, flags);
 926
 927         dma->tx_running = 0;
 928
 929         xmit->tail += dma->tx_size;
 930         xmit->tail &= UART_XMIT_SIZE - 1;
 931         p->port.icount.tx += dma->tx_size;
 932
 933         if (priv->delayed_restore) {
 934                 priv->delayed_restore = 0;
 935                 omap8250_restore_regs(p);
 936         }
 937
 938         if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
 939                 uart_write_wakeup(&p->port);
 940
 941         if (!uart_circ_empty(xmit) && !uart_tx_stopped(&p->port)) {
 942                 int ret;
 943
 944                 ret = omap_8250_tx_dma(p);
 945                 if (ret)
 946                         en_thri = true;
 947         } else if (p->capabilities & UART_CAP_RPM) {
 948                 en_thri = true;
 949         }
 950
 951         if (en_thri) {
 952                 dma->tx_err = 1;
 953                 serial8250_set_THRI(p);
 954         }
 955
 956         spin_unlock_irqrestore(&p->port.lock, flags);
 957 }
 958
 959 static int omap_8250_tx_dma(struct uart_8250_port *p)
 960 {
 961         struct uart_8250_dma            *dma = p->dma;
 962         struct omap8250_priv            *priv = p->port.private_data;
 963         struct circ_buf                 *xmit = &p->port.state->xmit;
 964         struct dma_async_tx_descriptor  *desc;
 965         unsigned int    skip_byte = 0;
 966         int ret;
 967
 968         if (dma->tx_running)
 969                 return 0;
 970         if (uart_tx_stopped(&p->port) || uart_circ_empty(xmit)) {
 971
 972                 /*
 973                  * Even if no data, we need to return an error for the two cases
 974                  * below so serial8250_tx_chars() is invoked and properly clears
 975                  * THRI and/or runtime suspend.
 976                  */
 977                 if (dma->tx_err || p->capabilities & UART_CAP_RPM) {
 978                         ret = -EBUSY;
 979                         goto err;
 980                 }
 981                 serial8250_clear_THRI(p);
 982                 return 0;
 983         }
 984
 985         dma->tx_size = CIRC_CNT_TO_END(xmit->head, xmit->tail, UART_XMIT_SIZE);
 986         if (priv->habit & OMAP_DMA_TX_KICK) {
 987                 u8 tx_lvl;
 988
 989                 /*
 990                  * We need to put the first byte into the FIFO in order to start
 991                  * the DMA transfer. For transfers smaller than four bytes we
 992                  * don't bother doing DMA at all. It seem not matter if there
 993                  * are still bytes in the FIFO from the last transfer (in case
 994                  * we got here directly from omap_8250_dma_tx_complete()). Bytes
 995                  * leaving the FIFO seem not to trigger the DMA transfer. It is
 996                  * really the byte that we put into the FIFO.
 997                  * If the FIFO is already full then we most likely got here from
 998                  * omap_8250_dma_tx_complete(). And this means the DMA engine
 999                  * just completed its work. We don't have to wait the complete
1000                  * 86us at 115200,8n1 but around 60us (not to mention lower
1001                  * baudrates). So in that case we take the interrupt and try
1002                  * again with an empty FIFO.
1003                  */
1004                 tx_lvl = serial_in(p, UART_OMAP_TX_LVL);
1005                 if (tx_lvl == p->tx_loadsz) {
1006                         ret = -EBUSY;
1007                         goto err;
1008                 }
1009                 if (dma->tx_size < 4) {
1010                         ret = -EINVAL;
1011                         goto err;
1012                 }
1013                 skip_byte = 1;
1014         }
1015
1016         desc = dmaengine_prep_slave_single(dma->txchan,
1017                         dma->tx_addr + xmit->tail + skip_byte,
1018                         dma->tx_size - skip_byte, DMA_MEM_TO_DEV,
1019                         DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1020         if (!desc) {
1021                 ret = -EBUSY;
1022                 goto err;
1023         }
1024
1025         dma->tx_running = 1;
1026
1027         desc->callback = omap_8250_dma_tx_complete;
1028         desc->callback_param = p;
1029
1030         dma->tx_cookie = dmaengine_submit(desc);
1031
1032         dma_sync_single_for_device(dma->txchan->device->dev, dma->tx_addr,
1033                                    UART_XMIT_SIZE, DMA_TO_DEVICE);
1034
1035         dma_async_issue_pending(dma->txchan);
1036         if (dma->tx_err)
1037                 dma->tx_err = 0;
1038
1039         serial8250_clear_THRI(p);
1040         if (skip_byte)
1041                 serial_out(p, UART_TX, xmit->buf[xmit->tail]);
1042         return 0;
1043 err:
1044         dma->tx_err = 1;
1045         return ret;
1046 }
1047
1048 static bool handle_rx_dma(struct uart_8250_port *up, unsigned int iir)
1049 {
1050         switch (iir & 0x3f) {
1051         case UART_IIR_RLSI:
1052         case UART_IIR_RX_TIMEOUT:
1053         case UART_IIR_RDI:
1054                 omap_8250_rx_dma_flush(up);
1055                 return true;
1056         }
1057         return omap_8250_rx_dma(up);
1058 }
1059
1060 static unsigned char omap_8250_handle_rx_dma(struct uart_8250_port *up,
1061                                              u8 iir, unsigned char status)
1062 {
1063         if ((status & (UART_LSR_DR | UART_LSR_BI)) &&
1064             (iir & UART_IIR_RDI)) {
1065                 if (handle_rx_dma(up, iir)) {
1066                         status = serial8250_rx_chars(up, status);
1067                         omap_8250_rx_dma(up);
1068                 }
1069         }
1070
1071         return status;
1072 }
1073
1074 static void am654_8250_handle_rx_dma(struct uart_8250_port *up, u8 iir,
1075                                      unsigned char status)
1076 {
1077         /*
1078          * Queue a new transfer if FIFO has data.
1079          */
1080         if ((status & (UART_LSR_DR | UART_LSR_BI)) &&
1081             (up->ier & UART_IER_RDI)) {
1082                 omap_8250_rx_dma(up);
1083                 serial_out(up, UART_OMAP_EFR2, UART_OMAP_EFR2_TIMEOUT_BEHAVE);
1084         } else if ((iir & 0x3f) == UART_IIR_RX_TIMEOUT) {
1085                 /*
1086                  * Disable RX timeout, read IIR to clear
1087                  * current timeout condition, clear EFR2 to
1088                  * periodic timeouts, re-enable interrupts.
1089                  */
1090                 up->ier &= ~(UART_IER_RLSI | UART_IER_RDI);
1091                 serial_out(up, UART_IER, up->ier);
1092                 omap_8250_rx_dma_flush(up);
1093                 serial_in(up, UART_IIR);
1094                 serial_out(up, UART_OMAP_EFR2, 0x0);
1095                 up->ier |= UART_IER_RLSI | UART_IER_RDI;
1096                 serial_out(up, UART_IER, up->ier);
1097         }
1098 }
1099
1100 /*
1101  * This is mostly serial8250_handle_irq(). We have a slightly different DMA
1102  * hoook for RX/TX and need different logic for them in the ISR. Therefore we
1103  * use the default routine in the non-DMA case and this one for with DMA.
1104  */
1105 static int omap_8250_dma_handle_irq(struct uart_port *port)
1106 {
1107         struct uart_8250_port *up = up_to_u8250p(port);
1108         struct omap8250_priv *priv = up->port.private_data;
1109         unsigned char status;
1110         unsigned long flags;
1111         u8 iir;
1112
1113         serial8250_rpm_get(up);
1114
1115         iir = serial_port_in(port, UART_IIR);
1116         if (iir & UART_IIR_NO_INT) {
1117                 serial8250_rpm_put(up);
1118                 return IRQ_HANDLED;
1119         }
1120
1121         spin_lock_irqsave(&port->lock, flags);
1122
1123         status = serial_port_in(port, UART_LSR);
1124
1125         if (priv->habit & UART_HAS_EFR2)
1126                 am654_8250_handle_rx_dma(up, iir, status);
1127         else
1128                 status = omap_8250_handle_rx_dma(up, iir, status);
1129
1130         serial8250_modem_status(up);
1131         if (status & UART_LSR_THRE && up->dma->tx_err) {
1132                 if (uart_tx_stopped(&up->port) ||
1133                     uart_circ_empty(&up->port.state->xmit)) {
1134                         up->dma->tx_err = 0;
1135                         serial8250_tx_chars(up);
1136                 } else  {
1137                         /*
1138                          * try again due to an earlier failer which
1139                          * might have been resolved by now.
1140                          */
1141                         if (omap_8250_tx_dma(up))
1142                                 serial8250_tx_chars(up);
1143                 }
1144         }
1145
1146         uart_unlock_and_check_sysrq(port, flags);
1147         serial8250_rpm_put(up);
1148         return 1;
1149 }
1150
1151 static bool the_no_dma_filter_fn(struct dma_chan *chan, void *param)
1152 {
1153         return false;
1154 }
1155
1156 #else
1157
1158 static inline int omap_8250_rx_dma(struct uart_8250_port *p)
1159 {
1160         return -EINVAL;
1161 }
1162 #endif
1163
1164 static int omap8250_no_handle_irq(struct uart_port *port)
1165 {
1166         /* IRQ has not been requested but handling irq? */
1167         WARN_ONCE(1, "Unexpected irq handling before port startup\n");
1168         return 0;
1169 }
1170
1171 static struct omap8250_dma_params am654_dma = {
1172         .rx_size = SZ_2K,
1173         .rx_trigger = 1,
1174         .tx_trigger = TX_TRIGGER,
1175 };
1176
1177 static struct omap8250_dma_params am33xx_dma = {
1178         .rx_size = RX_TRIGGER,
1179         .rx_trigger = RX_TRIGGER,
1180         .tx_trigger = TX_TRIGGER,
1181 };
1182
1183 static struct omap8250_platdata am654_platdata = {
1184         .dma_params     = &am654_dma,
1185         .habit          = UART_HAS_EFR2,
1186 };
1187
1188 static struct omap8250_platdata am33xx_platdata = {
1189         .dma_params     = &am33xx_dma,
1190         .habit          = OMAP_DMA_TX_KICK | UART_ERRATA_CLOCK_DISABLE,
1191 };
1192
1193 static struct omap8250_platdata omap4_platdata = {
1194         .dma_params     = &am33xx_dma,
1195         .habit          = UART_ERRATA_CLOCK_DISABLE,
1196 };
1197
1198 static const struct of_device_id omap8250_dt_ids[] = {
1199         { .compatible = "ti,am654-uart", .data = &am654_platdata, },
1200         { .compatible = "ti,omap2-uart" },
1201         { .compatible = "ti,omap3-uart" },
1202         { .compatible = "ti,omap4-uart", .data = &omap4_platdata, },
1203         { .compatible = "ti,am3352-uart", .data = &am33xx_platdata, },
1204         { .compatible = "ti,am4372-uart", .data = &am33xx_platdata, },
1205         { .compatible = "ti,dra742-uart", .data = &omap4_platdata, },
1206         {},
1207 };
1208 MODULE_DEVICE_TABLE(of, omap8250_dt_ids);
1209
1210 static int omap8250_probe(struct platform_device *pdev)
1211 {
1212         struct resource *regs = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1213         struct resource *irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
1214         struct device_node *np = pdev->dev.of_node;
1215         struct omap8250_priv *priv;
1216         const struct omap8250_platdata *pdata;
1217         struct uart_8250_port up;
1218         int ret;
1219         void __iomem *membase;
1220
1221         if (!regs || !irq) {
1222                 dev_err(&pdev->dev, "missing registers or irq\n");
1223                 return -EINVAL;
1224         }
1225
1226         priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
1227         if (!priv)
1228                 return -ENOMEM;
1229
1230         membase = devm_ioremap(&pdev->dev, regs->start,
1231                                        resource_size(regs));
1232         if (!membase)
1233                 return -ENODEV;
1234
1235         memset(&up, 0, sizeof(up));
1236         up.port.dev = &pdev->dev;
1237         up.port.mapbase = regs->start;
1238         up.port.membase = membase;
1239         up.port.irq = irq->start;
1240         /*
1241          * It claims to be 16C750 compatible however it is a little different.
1242          * It has EFR and has no FCR7_64byte bit. The AFE (which it claims to
1243          * have) is enabled via EFR instead of MCR. The type is set here 8250
1244          * just to get things going. UNKNOWN does not work for a few reasons and
1245          * we don't need our own type since we don't use 8250's set_termios()
1246          * or pm callback.
1247          */
1248         up.port.type = PORT_8250;
1249         up.port.iotype = UPIO_MEM;
1250         up.port.flags = UPF_FIXED_PORT | UPF_FIXED_TYPE | UPF_SOFT_FLOW |
1251                 UPF_HARD_FLOW;
1252         up.port.private_data = priv;
1253
1254         up.port.regshift = 2;
1255         up.port.fifosize = 64;
1256         up.tx_loadsz = 64;
1257         up.capabilities = UART_CAP_FIFO;
1258 #ifdef CONFIG_PM
1259         /*
1260          * Runtime PM is mostly transparent. However to do it right we need to a
1261          * TX empty interrupt before we can put the device to auto idle. So if
1262          * PM is not enabled we don't add that flag and can spare that one extra
1263          * interrupt in the TX path.
1264          */
1265         up.capabilities |= UART_CAP_RPM;
1266 #endif
1267         up.port.set_termios = omap_8250_set_termios;
1268         up.port.set_mctrl = omap8250_set_mctrl;
1269         up.port.pm = omap_8250_pm;
1270         up.port.startup = omap_8250_startup;
1271         up.port.shutdown = omap_8250_shutdown;
1272         up.port.throttle = omap_8250_throttle;
1273         up.port.unthrottle = omap_8250_unthrottle;
1274         up.port.rs485_config = serial8250_em485_config;
1275         up.rs485_start_tx = serial8250_em485_start_tx;
1276         up.rs485_stop_tx = serial8250_em485_stop_tx;
1277         up.port.has_sysrq = IS_ENABLED(CONFIG_SERIAL_8250_CONSOLE);
1278
1279         ret = of_alias_get_id(np, "serial");
1280         if (ret < 0) {
1281                 dev_err(&pdev->dev, "failed to get alias\n");
1282                 return ret;
1283         }
1284         up.port.line = ret;
1285
1286         if (of_property_read_u32(np, "clock-frequency", &up.port.uartclk)) {
1287                 struct clk *clk;
1288
1289                 clk = devm_clk_get(&pdev->dev, NULL);
1290                 if (IS_ERR(clk)) {
1291                         if (PTR_ERR(clk) == -EPROBE_DEFER)
1292                                 return -EPROBE_DEFER;
1293                 } else {
1294                         up.port.uartclk = clk_get_rate(clk);
1295                 }
1296         }
1297
1298         priv->wakeirq = irq_of_parse_and_map(np, 1);
1299
1300         pdata = of_device_get_match_data(&pdev->dev);
1301         if (pdata)
1302                 priv->habit |= pdata->habit;
1303
1304         if (!up.port.uartclk) {
1305                 up.port.uartclk = DEFAULT_CLK_SPEED;
1306                 dev_warn(&pdev->dev,
1307                          "No clock speed specified: using default: %d\n",
1308                          DEFAULT_CLK_SPEED);
1309         }
1310
1311         priv->latency = PM_QOS_CPU_LATENCY_DEFAULT_VALUE;
1312         priv->calc_latency = PM_QOS_CPU_LATENCY_DEFAULT_VALUE;
1313         cpu_latency_qos_add_request(&priv->pm_qos_request, priv->latency);
1314         INIT_WORK(&priv->qos_work, omap8250_uart_qos_work);
1315
1316         spin_lock_init(&priv->rx_dma_lock);
1317
1318         device_init_wakeup(&pdev->dev, true);
1319         pm_runtime_enable(&pdev->dev);
1320         pm_runtime_use_autosuspend(&pdev->dev);
1321
1322         /*
1323          * Disable runtime PM until autosuspend delay unless specifically
1324          * enabled by the user via sysfs. This is the historic way to
1325          * prevent an unsafe default policy with lossy characters on wake-up.
1326          * For serdev devices this is not needed, the policy can be managed by
1327          * the serdev driver.
1328          */
1329         if (!of_get_available_child_count(pdev->dev.of_node))
1330                 pm_runtime_set_autosuspend_delay(&pdev->dev, -1);
1331
1332         pm_runtime_irq_safe(&pdev->dev);
1333
1334         pm_runtime_get_sync(&pdev->dev);
1335
1336         omap_serial_fill_features_erratas(&up, priv);
1337         up.port.handle_irq = omap8250_no_handle_irq;
1338         priv->rx_trigger = RX_TRIGGER;
1339         priv->tx_trigger = TX_TRIGGER;
1340 #ifdef CONFIG_SERIAL_8250_DMA
1341         /*
1342          * Oh DMA support. If there are no DMA properties in the DT then
1343          * we will fall back to a generic DMA channel which does not
1344          * really work here. To ensure that we do not get a generic DMA
1345          * channel assigned, we have the the_no_dma_filter_fn() here.
1346          * To avoid "failed to request DMA" messages we check for DMA
1347          * properties in DT.
1348          */
1349         ret = of_property_count_strings(np, "dma-names");
1350         if (ret == 2) {
1351                 struct omap8250_dma_params *dma_params = NULL;
1352
1353                 up.dma = &priv->omap8250_dma;
1354                 up.dma->fn = the_no_dma_filter_fn;
1355                 up.dma->tx_dma = omap_8250_tx_dma;
1356                 up.dma->rx_dma = omap_8250_rx_dma;
1357                 if (pdata)
1358                         dma_params = pdata->dma_params;
1359
1360                 if (dma_params) {
1361                         up.dma->rx_size = dma_params->rx_size;
1362                         up.dma->rxconf.src_maxburst = dma_params->rx_trigger;
1363                         up.dma->txconf.dst_maxburst = dma_params->tx_trigger;
1364                         priv->rx_trigger = dma_params->rx_trigger;
1365                         priv->tx_trigger = dma_params->tx_trigger;
1366                 } else {
1367                         up.dma->rx_size = RX_TRIGGER;
1368                         up.dma->rxconf.src_maxburst = RX_TRIGGER;
1369                         up.dma->txconf.dst_maxburst = TX_TRIGGER;
1370                 }
1371         }
1372 #endif
1373         ret = serial8250_register_8250_port(&up);
1374         if (ret < 0) {
1375                 dev_err(&pdev->dev, "unable to register 8250 port\n");
1376                 goto err;
1377         }
1378         priv->line = ret;
1379         platform_set_drvdata(pdev, priv);
1380         pm_runtime_mark_last_busy(&pdev->dev);
1381         pm_runtime_put_autosuspend(&pdev->dev);
1382         return 0;
1383 err:
1384         pm_runtime_dont_use_autosuspend(&pdev->dev);
1385         pm_runtime_put_sync(&pdev->dev);
1386         pm_runtime_disable(&pdev->dev);
1387         return ret;
1388 }
1389
1390 static int omap8250_remove(struct platform_device *pdev)
1391 {
1392         struct omap8250_priv *priv = platform_get_drvdata(pdev);
1393
1394         pm_runtime_dont_use_autosuspend(&pdev->dev);
1395         pm_runtime_put_sync(&pdev->dev);
1396         pm_runtime_disable(&pdev->dev);
1397         serial8250_unregister_port(priv->line);
1398         cpu_latency_qos_remove_request(&priv->pm_qos_request);
1399         device_init_wakeup(&pdev->dev, false);
1400         return 0;
1401 }
1402
1403 #ifdef CONFIG_PM_SLEEP
1404 static int omap8250_prepare(struct device *dev)
1405 {
1406         struct omap8250_priv *priv = dev_get_drvdata(dev);
1407
1408         if (!priv)
1409                 return 0;
1410         priv->is_suspending = true;
1411         return 0;
1412 }
1413
1414 static void omap8250_complete(struct device *dev)
1415 {
1416         struct omap8250_priv *priv = dev_get_drvdata(dev);
1417
1418         if (!priv)
1419                 return;
1420         priv->is_suspending = false;
1421 }
1422
1423 static int omap8250_suspend(struct device *dev)
1424 {
1425         struct omap8250_priv *priv = dev_get_drvdata(dev);
1426         struct uart_8250_port *up = serial8250_get_port(priv->line);
1427
1428         serial8250_suspend_port(priv->line);
1429
1430         pm_runtime_get_sync(dev);
1431         if (!device_may_wakeup(dev))
1432                 priv->wer = 0;
1433         serial_out(up, UART_OMAP_WER, priv->wer);
1434         pm_runtime_mark_last_busy(dev);
1435         pm_runtime_put_autosuspend(dev);
1436
1437         flush_work(&priv->qos_work);
1438         return 0;
1439 }
1440
1441 static int omap8250_resume(struct device *dev)
1442 {
1443         struct omap8250_priv *priv = dev_get_drvdata(dev);
1444
1445         serial8250_resume_port(priv->line);
1446         return 0;
1447 }
1448 #else
1449 #define omap8250_prepare NULL
1450 #define omap8250_complete NULL
1451 #endif
1452
1453 #ifdef CONFIG_PM
1454 static int omap8250_lost_context(struct uart_8250_port *up)
1455 {
1456         u32 val;
1457
1458         val = serial_in(up, UART_OMAP_SCR);
1459         /*
1460          * If we lose context, then SCR is set to its reset value of zero.
1461          * After set_termios() we set bit 3 of SCR (TX_EMPTY_CTL_IT) to 1,
1462          * among other bits, to never set the register back to zero again.
1463          */
1464         if (!val)
1465                 return 1;
1466         return 0;
1467 }
1468
1469 /* TODO: in future, this should happen via API in drivers/reset/ */
1470 static int omap8250_soft_reset(struct device *dev)
1471 {
1472         struct omap8250_priv *priv = dev_get_drvdata(dev);
1473         struct uart_8250_port *up = serial8250_get_port(priv->line);
1474         int timeout = 100;
1475         int sysc;
1476         int syss;
1477
1478         /*
1479          * At least on omap4, unused uarts may not idle after reset without
1480          * a basic scr dma configuration even with no dma in use. The
1481          * module clkctrl status bits will be 1 instead of 3 blocking idle
1482          * for the whole clockdomain. The softreset below will clear scr,
1483          * and we restore it on resume so this is safe to do on all SoCs
1484          * needing omap8250_soft_reset() quirk. Do it in two writes as
1485          * recommended in the comment for omap8250_update_scr().
1486          */
1487         serial_out(up, UART_OMAP_SCR, OMAP_UART_SCR_DMAMODE_1);
1488         serial_out(up, UART_OMAP_SCR,
1489                    OMAP_UART_SCR_DMAMODE_1 | OMAP_UART_SCR_DMAMODE_CTL);
1490
1491         sysc = serial_in(up, UART_OMAP_SYSC);
1492
1493         /* softreset the UART */
1494         sysc |= OMAP_UART_SYSC_SOFTRESET;
1495         serial_out(up, UART_OMAP_SYSC, sysc);
1496
1497         /* By experiments, 1us enough for reset complete on AM335x */
1498         do {
1499                 udelay(1);
1500                 syss = serial_in(up, UART_OMAP_SYSS);
1501         } while (--timeout && !(syss & OMAP_UART_SYSS_RESETDONE));
1502
1503         if (!timeout) {
1504                 dev_err(dev, "timed out waiting for reset done\n");
1505                 return -ETIMEDOUT;
1506         }
1507
1508         return 0;
1509 }
1510
1511 static int omap8250_runtime_suspend(struct device *dev)
1512 {
1513         struct omap8250_priv *priv = dev_get_drvdata(dev);
1514         struct uart_8250_port *up;
1515
1516         /* In case runtime-pm tries this before we are setup */
1517         if (!priv)
1518                 return 0;
1519
1520         up = serial8250_get_port(priv->line);
1521         /*
1522          * When using 'no_console_suspend', the console UART must not be
1523          * suspended. Since driver suspend is managed by runtime suspend,
1524          * preventing runtime suspend (by returning error) will keep device
1525          * active during suspend.
1526          */
1527         if (priv->is_suspending && !console_suspend_enabled) {
1528                 if (uart_console(&up->port))
1529                         return -EBUSY;
1530         }
1531
1532         if (priv->habit & UART_ERRATA_CLOCK_DISABLE) {
1533                 int ret;
1534
1535                 ret = omap8250_soft_reset(dev);
1536                 if (ret)
1537                         return ret;
1538
1539                 /* Restore to UART mode after reset (for wakeup) */
1540                 omap8250_update_mdr1(up, priv);
1541                 /* Restore wakeup enable register */
1542                 serial_out(up, UART_OMAP_WER, priv->wer);
1543         }
1544
1545         if (up->dma && up->dma->rxchan)
1546                 omap_8250_rx_dma_flush(up);
1547
1548         priv->latency = PM_QOS_CPU_LATENCY_DEFAULT_VALUE;
1549         schedule_work(&priv->qos_work);
1550
1551         return 0;
1552 }
1553
1554 static int omap8250_runtime_resume(struct device *dev)
1555 {
1556         struct omap8250_priv *priv = dev_get_drvdata(dev);
1557         struct uart_8250_port *up;
1558
1559         /* In case runtime-pm tries this before we are setup */
1560         if (!priv)
1561                 return 0;
1562
1563         up = serial8250_get_port(priv->line);
1564
1565         if (omap8250_lost_context(up))
1566                 omap8250_restore_regs(up);
1567
1568         if (up->dma && up->dma->rxchan && !(priv->habit & UART_HAS_EFR2))
1569                 omap_8250_rx_dma(up);
1570
1571         priv->latency = priv->calc_latency;
1572         schedule_work(&priv->qos_work);
1573         return 0;
1574 }
1575 #endif
1576
1577 #ifdef CONFIG_SERIAL_8250_OMAP_TTYO_FIXUP
1578 static int __init omap8250_console_fixup(void)
1579 {
1580         char *omap_str;
1581         char *options;
1582         u8 idx;
1583
1584         if (strstr(boot_command_line, "console=ttyS"))
1585                 /* user set a ttyS based name for the console */
1586                 return 0;
1587
1588         omap_str = strstr(boot_command_line, "console=ttyO");
1589         if (!omap_str)
1590                 /* user did not set ttyO based console, so we don't care */
1591                 return 0;
1592
1593         omap_str += 12;
1594         if ('0' <= *omap_str && *omap_str <= '9')
1595                 idx = *omap_str - '0';
1596         else
1597                 return 0;
1598
1599         omap_str++;
1600         if (omap_str[0] == ',') {
1601                 omap_str++;
1602                 options = omap_str;
1603         } else {
1604                 options = NULL;
1605         }
1606
1607         add_preferred_console("ttyS", idx, options);
1608         pr_err("WARNING: Your 'console=ttyO%d' has been replaced by 'ttyS%d'\n",
1609                idx, idx);
1610         pr_err("This ensures that you still see kernel messages. Please\n");
1611         pr_err("update your kernel commandline.\n");
1612         return 0;
1613 }
1614 console_initcall(omap8250_console_fixup);
1615 #endif
1616
1617 static const struct dev_pm_ops omap8250_dev_pm_ops = {
1618         SET_SYSTEM_SLEEP_PM_OPS(omap8250_suspend, omap8250_resume)
1619         SET_RUNTIME_PM_OPS(omap8250_runtime_suspend,
1620                            omap8250_runtime_resume, NULL)
1621         .prepare        = omap8250_prepare,
1622         .complete       = omap8250_complete,
1623 };
1624
1625 static struct platform_driver omap8250_platform_driver = {
1626         .driver = {
1627                 .name           = "omap8250",
1628                 .pm             = &omap8250_dev_pm_ops,
1629                 .of_match_table = omap8250_dt_ids,
1630         },
1631         .probe                  = omap8250_probe,
1632         .remove                 = omap8250_remove,
1633 };
1634 module_platform_driver(omap8250_platform_driver);
1635
1636 MODULE_AUTHOR("Sebastian Andrzej Siewior");
1637 MODULE_DESCRIPTION("OMAP 8250 Driver");
1638 MODULE_LICENSE("GPL v2");