1 // SPDX-License-Identifier: GPL-2.0+
3 * Driver for AMBA serial ports
5 * Based on drivers/char/serial.c, by Linus Torvalds, Theodore Ts'o.
7 * Copyright 1999 ARM Limited
8 * Copyright (C) 2000 Deep Blue Solutions Ltd.
9 * Copyright (C) 2010 ST-Ericsson SA
11 * This is a generic driver for ARM AMBA-type serial ports. They
12 * have a lot of 16550-like features, but are not register compatible.
13 * Note that although they do have CTS, DCD and DSR inputs, they do
14 * not have an RI input, nor do they have DTR or RTS outputs. If
15 * required, these have to be supplied via some other means (eg, GPIO)
16 * and hooked into this driver.
19 #include <linux/module.h>
20 #include <linux/ioport.h>
21 #include <linux/init.h>
22 #include <linux/console.h>
23 #include <linux/sysrq.h>
24 #include <linux/device.h>
25 #include <linux/tty.h>
26 #include <linux/tty_flip.h>
27 #include <linux/serial_core.h>
28 #include <linux/serial.h>
29 #include <linux/amba/bus.h>
30 #include <linux/amba/serial.h>
31 #include <linux/clk.h>
32 #include <linux/slab.h>
33 #include <linux/dmaengine.h>
34 #include <linux/dma-mapping.h>
35 #include <linux/scatterlist.h>
36 #include <linux/delay.h>
37 #include <linux/types.h>
39 #include <linux/of_device.h>
40 #include <linux/pinctrl/consumer.h>
41 #include <linux/sizes.h>
43 #include <linux/acpi.h>
45 #include "amba-pl011.h"
49 #define SERIAL_AMBA_MAJOR 204
50 #define SERIAL_AMBA_MINOR 64
51 #define SERIAL_AMBA_NR UART_NR
53 #define AMBA_ISR_PASS_LIMIT 256
55 #define UART_DR_ERROR (UART011_DR_OE|UART011_DR_BE|UART011_DR_PE|UART011_DR_FE)
56 #define UART_DUMMY_DR_RX (1 << 16)
58 static u16 pl011_std_offsets[REG_ARRAY_SIZE] = {
59 [REG_DR] = UART01x_DR,
60 [REG_FR] = UART01x_FR,
61 [REG_LCRH_RX] = UART011_LCRH,
62 [REG_LCRH_TX] = UART011_LCRH,
63 [REG_IBRD] = UART011_IBRD,
64 [REG_FBRD] = UART011_FBRD,
65 [REG_CR] = UART011_CR,
66 [REG_IFLS] = UART011_IFLS,
67 [REG_IMSC] = UART011_IMSC,
68 [REG_RIS] = UART011_RIS,
69 [REG_MIS] = UART011_MIS,
70 [REG_ICR] = UART011_ICR,
71 [REG_DMACR] = UART011_DMACR,
74 /* There is by now at least one vendor with differing details, so handle it */
76 const u16 *reg_offset;
86 bool cts_event_workaround;
90 unsigned int (*get_fifosize)(struct amba_device *dev);
93 static unsigned int get_fifosize_arm(struct amba_device *dev)
95 return amba_rev(dev) < 3 ? 16 : 32;
98 static struct vendor_data vendor_arm = {
99 .reg_offset = pl011_std_offsets,
100 .ifls = UART011_IFLS_RX4_8|UART011_IFLS_TX4_8,
101 .fr_busy = UART01x_FR_BUSY,
102 .fr_dsr = UART01x_FR_DSR,
103 .fr_cts = UART01x_FR_CTS,
104 .fr_ri = UART011_FR_RI,
105 .oversampling = false,
106 .dma_threshold = false,
107 .cts_event_workaround = false,
108 .always_enabled = false,
109 .fixed_options = false,
110 .get_fifosize = get_fifosize_arm,
113 static const struct vendor_data vendor_sbsa = {
114 .reg_offset = pl011_std_offsets,
115 .fr_busy = UART01x_FR_BUSY,
116 .fr_dsr = UART01x_FR_DSR,
117 .fr_cts = UART01x_FR_CTS,
118 .fr_ri = UART011_FR_RI,
120 .oversampling = false,
121 .dma_threshold = false,
122 .cts_event_workaround = false,
123 .always_enabled = true,
124 .fixed_options = true,
127 #ifdef CONFIG_ACPI_SPCR_TABLE
128 static const struct vendor_data vendor_qdt_qdf2400_e44 = {
129 .reg_offset = pl011_std_offsets,
130 .fr_busy = UART011_FR_TXFE,
131 .fr_dsr = UART01x_FR_DSR,
132 .fr_cts = UART01x_FR_CTS,
133 .fr_ri = UART011_FR_RI,
134 .inv_fr = UART011_FR_TXFE,
136 .oversampling = false,
137 .dma_threshold = false,
138 .cts_event_workaround = false,
139 .always_enabled = true,
140 .fixed_options = true,
144 static u16 pl011_st_offsets[REG_ARRAY_SIZE] = {
145 [REG_DR] = UART01x_DR,
146 [REG_ST_DMAWM] = ST_UART011_DMAWM,
147 [REG_ST_TIMEOUT] = ST_UART011_TIMEOUT,
148 [REG_FR] = UART01x_FR,
149 [REG_LCRH_RX] = ST_UART011_LCRH_RX,
150 [REG_LCRH_TX] = ST_UART011_LCRH_TX,
151 [REG_IBRD] = UART011_IBRD,
152 [REG_FBRD] = UART011_FBRD,
153 [REG_CR] = UART011_CR,
154 [REG_IFLS] = UART011_IFLS,
155 [REG_IMSC] = UART011_IMSC,
156 [REG_RIS] = UART011_RIS,
157 [REG_MIS] = UART011_MIS,
158 [REG_ICR] = UART011_ICR,
159 [REG_DMACR] = UART011_DMACR,
160 [REG_ST_XFCR] = ST_UART011_XFCR,
161 [REG_ST_XON1] = ST_UART011_XON1,
162 [REG_ST_XON2] = ST_UART011_XON2,
163 [REG_ST_XOFF1] = ST_UART011_XOFF1,
164 [REG_ST_XOFF2] = ST_UART011_XOFF2,
165 [REG_ST_ITCR] = ST_UART011_ITCR,
166 [REG_ST_ITIP] = ST_UART011_ITIP,
167 [REG_ST_ABCR] = ST_UART011_ABCR,
168 [REG_ST_ABIMSC] = ST_UART011_ABIMSC,
171 static unsigned int get_fifosize_st(struct amba_device *dev)
176 static struct vendor_data vendor_st = {
177 .reg_offset = pl011_st_offsets,
178 .ifls = UART011_IFLS_RX_HALF|UART011_IFLS_TX_HALF,
179 .fr_busy = UART01x_FR_BUSY,
180 .fr_dsr = UART01x_FR_DSR,
181 .fr_cts = UART01x_FR_CTS,
182 .fr_ri = UART011_FR_RI,
183 .oversampling = true,
184 .dma_threshold = true,
185 .cts_event_workaround = true,
186 .always_enabled = false,
187 .fixed_options = false,
188 .get_fifosize = get_fifosize_st,
191 static const u16 pl011_zte_offsets[REG_ARRAY_SIZE] = {
192 [REG_DR] = ZX_UART011_DR,
193 [REG_FR] = ZX_UART011_FR,
194 [REG_LCRH_RX] = ZX_UART011_LCRH,
195 [REG_LCRH_TX] = ZX_UART011_LCRH,
196 [REG_IBRD] = ZX_UART011_IBRD,
197 [REG_FBRD] = ZX_UART011_FBRD,
198 [REG_CR] = ZX_UART011_CR,
199 [REG_IFLS] = ZX_UART011_IFLS,
200 [REG_IMSC] = ZX_UART011_IMSC,
201 [REG_RIS] = ZX_UART011_RIS,
202 [REG_MIS] = ZX_UART011_MIS,
203 [REG_ICR] = ZX_UART011_ICR,
204 [REG_DMACR] = ZX_UART011_DMACR,
207 static unsigned int get_fifosize_zte(struct amba_device *dev)
212 static struct vendor_data vendor_zte = {
213 .reg_offset = pl011_zte_offsets,
215 .ifls = UART011_IFLS_RX4_8|UART011_IFLS_TX4_8,
216 .fr_busy = ZX_UART01x_FR_BUSY,
217 .fr_dsr = ZX_UART01x_FR_DSR,
218 .fr_cts = ZX_UART01x_FR_CTS,
219 .fr_ri = ZX_UART011_FR_RI,
220 .get_fifosize = get_fifosize_zte,
223 /* Deals with DMA transactions */
226 struct scatterlist sg;
230 struct pl011_dmarx_data {
231 struct dma_chan *chan;
232 struct completion complete;
234 struct pl011_sgbuf sgbuf_a;
235 struct pl011_sgbuf sgbuf_b;
238 struct timer_list timer;
239 unsigned int last_residue;
240 unsigned long last_jiffies;
242 unsigned int poll_rate;
243 unsigned int poll_timeout;
246 struct pl011_dmatx_data {
247 struct dma_chan *chan;
248 struct scatterlist sg;
254 * We wrap our port structure around the generic uart_port.
256 struct uart_amba_port {
257 struct uart_port port;
258 const u16 *reg_offset;
260 const struct vendor_data *vendor;
261 unsigned int dmacr; /* dma control reg */
262 unsigned int im; /* interrupt mask */
263 unsigned int old_status;
264 unsigned int fifosize; /* vendor-specific */
265 unsigned int old_cr; /* state during shutdown */
266 unsigned int fixed_baud; /* vendor-set fixed baud rate */
268 bool rs485_tx_started;
269 unsigned int rs485_tx_drain_interval; /* usecs */
270 #ifdef CONFIG_DMA_ENGINE
274 struct pl011_dmarx_data dmarx;
275 struct pl011_dmatx_data dmatx;
280 static unsigned int pl011_tx_empty(struct uart_port *port);
282 static unsigned int pl011_reg_to_offset(const struct uart_amba_port *uap,
285 return uap->reg_offset[reg];
288 static unsigned int pl011_read(const struct uart_amba_port *uap,
291 void __iomem *addr = uap->port.membase + pl011_reg_to_offset(uap, reg);
293 return (uap->port.iotype == UPIO_MEM32) ?
294 readl_relaxed(addr) : readw_relaxed(addr);
297 static void pl011_write(unsigned int val, const struct uart_amba_port *uap,
300 void __iomem *addr = uap->port.membase + pl011_reg_to_offset(uap, reg);
302 if (uap->port.iotype == UPIO_MEM32)
303 writel_relaxed(val, addr);
305 writew_relaxed(val, addr);
309 * Reads up to 256 characters from the FIFO or until it's empty and
310 * inserts them into the TTY layer. Returns the number of characters
311 * read from the FIFO.
313 static int pl011_fifo_to_tty(struct uart_amba_port *uap)
315 unsigned int ch, flag, fifotaken;
319 for (fifotaken = 0; fifotaken != 256; fifotaken++) {
320 status = pl011_read(uap, REG_FR);
321 if (status & UART01x_FR_RXFE)
324 /* Take chars from the FIFO and update status */
325 ch = pl011_read(uap, REG_DR) | UART_DUMMY_DR_RX;
327 uap->port.icount.rx++;
329 if (unlikely(ch & UART_DR_ERROR)) {
330 if (ch & UART011_DR_BE) {
331 ch &= ~(UART011_DR_FE | UART011_DR_PE);
332 uap->port.icount.brk++;
333 if (uart_handle_break(&uap->port))
335 } else if (ch & UART011_DR_PE)
336 uap->port.icount.parity++;
337 else if (ch & UART011_DR_FE)
338 uap->port.icount.frame++;
339 if (ch & UART011_DR_OE)
340 uap->port.icount.overrun++;
342 ch &= uap->port.read_status_mask;
344 if (ch & UART011_DR_BE)
346 else if (ch & UART011_DR_PE)
348 else if (ch & UART011_DR_FE)
352 spin_unlock(&uap->port.lock);
353 sysrq = uart_handle_sysrq_char(&uap->port, ch & 255);
354 spin_lock(&uap->port.lock);
357 uart_insert_char(&uap->port, ch, UART011_DR_OE, ch, flag);
365 * All the DMA operation mode stuff goes inside this ifdef.
366 * This assumes that you have a generic DMA device interface,
367 * no custom DMA interfaces are supported.
369 #ifdef CONFIG_DMA_ENGINE
371 #define PL011_DMA_BUFFER_SIZE PAGE_SIZE
373 static int pl011_sgbuf_init(struct dma_chan *chan, struct pl011_sgbuf *sg,
374 enum dma_data_direction dir)
378 sg->buf = dma_alloc_coherent(chan->device->dev,
379 PL011_DMA_BUFFER_SIZE, &dma_addr, GFP_KERNEL);
383 sg_init_table(&sg->sg, 1);
384 sg_set_page(&sg->sg, phys_to_page(dma_addr),
385 PL011_DMA_BUFFER_SIZE, offset_in_page(dma_addr));
386 sg_dma_address(&sg->sg) = dma_addr;
387 sg_dma_len(&sg->sg) = PL011_DMA_BUFFER_SIZE;
392 static void pl011_sgbuf_free(struct dma_chan *chan, struct pl011_sgbuf *sg,
393 enum dma_data_direction dir)
396 dma_free_coherent(chan->device->dev,
397 PL011_DMA_BUFFER_SIZE, sg->buf,
398 sg_dma_address(&sg->sg));
402 static void pl011_dma_probe(struct uart_amba_port *uap)
404 /* DMA is the sole user of the platform data right now */
405 struct amba_pl011_data *plat = dev_get_platdata(uap->port.dev);
406 struct device *dev = uap->port.dev;
407 struct dma_slave_config tx_conf = {
408 .dst_addr = uap->port.mapbase +
409 pl011_reg_to_offset(uap, REG_DR),
410 .dst_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE,
411 .direction = DMA_MEM_TO_DEV,
412 .dst_maxburst = uap->fifosize >> 1,
415 struct dma_chan *chan;
418 uap->dma_probed = true;
419 chan = dma_request_chan(dev, "tx");
421 if (PTR_ERR(chan) == -EPROBE_DEFER) {
422 uap->dma_probed = false;
426 /* We need platform data */
427 if (!plat || !plat->dma_filter) {
428 dev_info(uap->port.dev, "no DMA platform data\n");
432 /* Try to acquire a generic DMA engine slave TX channel */
434 dma_cap_set(DMA_SLAVE, mask);
436 chan = dma_request_channel(mask, plat->dma_filter,
439 dev_err(uap->port.dev, "no TX DMA channel!\n");
444 dmaengine_slave_config(chan, &tx_conf);
445 uap->dmatx.chan = chan;
447 dev_info(uap->port.dev, "DMA channel TX %s\n",
448 dma_chan_name(uap->dmatx.chan));
450 /* Optionally make use of an RX channel as well */
451 chan = dma_request_slave_channel(dev, "rx");
453 if (!chan && plat && plat->dma_rx_param) {
454 chan = dma_request_channel(mask, plat->dma_filter, plat->dma_rx_param);
457 dev_err(uap->port.dev, "no RX DMA channel!\n");
463 struct dma_slave_config rx_conf = {
464 .src_addr = uap->port.mapbase +
465 pl011_reg_to_offset(uap, REG_DR),
466 .src_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE,
467 .direction = DMA_DEV_TO_MEM,
468 .src_maxburst = uap->fifosize >> 2,
471 struct dma_slave_caps caps;
474 * Some DMA controllers provide information on their capabilities.
475 * If the controller does, check for suitable residue processing
476 * otherwise assime all is well.
478 if (0 == dma_get_slave_caps(chan, &caps)) {
479 if (caps.residue_granularity ==
480 DMA_RESIDUE_GRANULARITY_DESCRIPTOR) {
481 dma_release_channel(chan);
482 dev_info(uap->port.dev,
483 "RX DMA disabled - no residue processing\n");
487 dmaengine_slave_config(chan, &rx_conf);
488 uap->dmarx.chan = chan;
490 uap->dmarx.auto_poll_rate = false;
491 if (plat && plat->dma_rx_poll_enable) {
492 /* Set poll rate if specified. */
493 if (plat->dma_rx_poll_rate) {
494 uap->dmarx.auto_poll_rate = false;
495 uap->dmarx.poll_rate = plat->dma_rx_poll_rate;
498 * 100 ms defaults to poll rate if not
499 * specified. This will be adjusted with
500 * the baud rate at set_termios.
502 uap->dmarx.auto_poll_rate = true;
503 uap->dmarx.poll_rate = 100;
505 /* 3 secs defaults poll_timeout if not specified. */
506 if (plat->dma_rx_poll_timeout)
507 uap->dmarx.poll_timeout =
508 plat->dma_rx_poll_timeout;
510 uap->dmarx.poll_timeout = 3000;
511 } else if (!plat && dev->of_node) {
512 uap->dmarx.auto_poll_rate = of_property_read_bool(
513 dev->of_node, "auto-poll");
514 if (uap->dmarx.auto_poll_rate) {
517 if (0 == of_property_read_u32(dev->of_node,
519 uap->dmarx.poll_rate = x;
521 uap->dmarx.poll_rate = 100;
522 if (0 == of_property_read_u32(dev->of_node,
523 "poll-timeout-ms", &x))
524 uap->dmarx.poll_timeout = x;
526 uap->dmarx.poll_timeout = 3000;
529 dev_info(uap->port.dev, "DMA channel RX %s\n",
530 dma_chan_name(uap->dmarx.chan));
534 static void pl011_dma_remove(struct uart_amba_port *uap)
537 dma_release_channel(uap->dmatx.chan);
539 dma_release_channel(uap->dmarx.chan);
542 /* Forward declare these for the refill routine */
543 static int pl011_dma_tx_refill(struct uart_amba_port *uap);
544 static void pl011_start_tx_pio(struct uart_amba_port *uap);
547 * The current DMA TX buffer has been sent.
548 * Try to queue up another DMA buffer.
550 static void pl011_dma_tx_callback(void *data)
552 struct uart_amba_port *uap = data;
553 struct pl011_dmatx_data *dmatx = &uap->dmatx;
557 spin_lock_irqsave(&uap->port.lock, flags);
558 if (uap->dmatx.queued)
559 dma_unmap_sg(dmatx->chan->device->dev, &dmatx->sg, 1,
563 uap->dmacr = dmacr & ~UART011_TXDMAE;
564 pl011_write(uap->dmacr, uap, REG_DMACR);
567 * If TX DMA was disabled, it means that we've stopped the DMA for
568 * some reason (eg, XOFF received, or we want to send an X-char.)
570 * Note: we need to be careful here of a potential race between DMA
571 * and the rest of the driver - if the driver disables TX DMA while
572 * a TX buffer completing, we must update the tx queued status to
573 * get further refills (hence we check dmacr).
575 if (!(dmacr & UART011_TXDMAE) || uart_tx_stopped(&uap->port) ||
576 uart_circ_empty(&uap->port.state->xmit)) {
577 uap->dmatx.queued = false;
578 spin_unlock_irqrestore(&uap->port.lock, flags);
582 if (pl011_dma_tx_refill(uap) <= 0)
584 * We didn't queue a DMA buffer for some reason, but we
585 * have data pending to be sent. Re-enable the TX IRQ.
587 pl011_start_tx_pio(uap);
589 spin_unlock_irqrestore(&uap->port.lock, flags);
593 * Try to refill the TX DMA buffer.
594 * Locking: called with port lock held and IRQs disabled.
596 * 1 if we queued up a TX DMA buffer.
597 * 0 if we didn't want to handle this by DMA
600 static int pl011_dma_tx_refill(struct uart_amba_port *uap)
602 struct pl011_dmatx_data *dmatx = &uap->dmatx;
603 struct dma_chan *chan = dmatx->chan;
604 struct dma_device *dma_dev = chan->device;
605 struct dma_async_tx_descriptor *desc;
606 struct circ_buf *xmit = &uap->port.state->xmit;
610 * Try to avoid the overhead involved in using DMA if the
611 * transaction fits in the first half of the FIFO, by using
612 * the standard interrupt handling. This ensures that we
613 * issue a uart_write_wakeup() at the appropriate time.
615 count = uart_circ_chars_pending(xmit);
616 if (count < (uap->fifosize >> 1)) {
617 uap->dmatx.queued = false;
622 * Bodge: don't send the last character by DMA, as this
623 * will prevent XON from notifying us to restart DMA.
627 /* Else proceed to copy the TX chars to the DMA buffer and fire DMA */
628 if (count > PL011_DMA_BUFFER_SIZE)
629 count = PL011_DMA_BUFFER_SIZE;
631 if (xmit->tail < xmit->head)
632 memcpy(&dmatx->buf[0], &xmit->buf[xmit->tail], count);
634 size_t first = UART_XMIT_SIZE - xmit->tail;
639 second = count - first;
641 memcpy(&dmatx->buf[0], &xmit->buf[xmit->tail], first);
643 memcpy(&dmatx->buf[first], &xmit->buf[0], second);
646 dmatx->sg.length = count;
648 if (dma_map_sg(dma_dev->dev, &dmatx->sg, 1, DMA_TO_DEVICE) != 1) {
649 uap->dmatx.queued = false;
650 dev_dbg(uap->port.dev, "unable to map TX DMA\n");
654 desc = dmaengine_prep_slave_sg(chan, &dmatx->sg, 1, DMA_MEM_TO_DEV,
655 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
657 dma_unmap_sg(dma_dev->dev, &dmatx->sg, 1, DMA_TO_DEVICE);
658 uap->dmatx.queued = false;
660 * If DMA cannot be used right now, we complete this
661 * transaction via IRQ and let the TTY layer retry.
663 dev_dbg(uap->port.dev, "TX DMA busy\n");
667 /* Some data to go along to the callback */
668 desc->callback = pl011_dma_tx_callback;
669 desc->callback_param = uap;
671 /* All errors should happen at prepare time */
672 dmaengine_submit(desc);
674 /* Fire the DMA transaction */
675 dma_dev->device_issue_pending(chan);
677 uap->dmacr |= UART011_TXDMAE;
678 pl011_write(uap->dmacr, uap, REG_DMACR);
679 uap->dmatx.queued = true;
682 * Now we know that DMA will fire, so advance the ring buffer
683 * with the stuff we just dispatched.
685 xmit->tail = (xmit->tail + count) & (UART_XMIT_SIZE - 1);
686 uap->port.icount.tx += count;
688 if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
689 uart_write_wakeup(&uap->port);
695 * We received a transmit interrupt without a pending X-char but with
696 * pending characters.
697 * Locking: called with port lock held and IRQs disabled.
699 * false if we want to use PIO to transmit
700 * true if we queued a DMA buffer
702 static bool pl011_dma_tx_irq(struct uart_amba_port *uap)
704 if (!uap->using_tx_dma)
708 * If we already have a TX buffer queued, but received a
709 * TX interrupt, it will be because we've just sent an X-char.
710 * Ensure the TX DMA is enabled and the TX IRQ is disabled.
712 if (uap->dmatx.queued) {
713 uap->dmacr |= UART011_TXDMAE;
714 pl011_write(uap->dmacr, uap, REG_DMACR);
715 uap->im &= ~UART011_TXIM;
716 pl011_write(uap->im, uap, REG_IMSC);
721 * We don't have a TX buffer queued, so try to queue one.
722 * If we successfully queued a buffer, mask the TX IRQ.
724 if (pl011_dma_tx_refill(uap) > 0) {
725 uap->im &= ~UART011_TXIM;
726 pl011_write(uap->im, uap, REG_IMSC);
733 * Stop the DMA transmit (eg, due to received XOFF).
734 * Locking: called with port lock held and IRQs disabled.
736 static inline void pl011_dma_tx_stop(struct uart_amba_port *uap)
738 if (uap->dmatx.queued) {
739 uap->dmacr &= ~UART011_TXDMAE;
740 pl011_write(uap->dmacr, uap, REG_DMACR);
745 * Try to start a DMA transmit, or in the case of an XON/OFF
746 * character queued for send, try to get that character out ASAP.
747 * Locking: called with port lock held and IRQs disabled.
749 * false if we want the TX IRQ to be enabled
750 * true if we have a buffer queued
752 static inline bool pl011_dma_tx_start(struct uart_amba_port *uap)
756 if (!uap->using_tx_dma)
759 if (!uap->port.x_char) {
760 /* no X-char, try to push chars out in DMA mode */
763 if (!uap->dmatx.queued) {
764 if (pl011_dma_tx_refill(uap) > 0) {
765 uap->im &= ~UART011_TXIM;
766 pl011_write(uap->im, uap, REG_IMSC);
769 } else if (!(uap->dmacr & UART011_TXDMAE)) {
770 uap->dmacr |= UART011_TXDMAE;
771 pl011_write(uap->dmacr, uap, REG_DMACR);
777 * We have an X-char to send. Disable DMA to prevent it loading
778 * the TX fifo, and then see if we can stuff it into the FIFO.
781 uap->dmacr &= ~UART011_TXDMAE;
782 pl011_write(uap->dmacr, uap, REG_DMACR);
784 if (pl011_read(uap, REG_FR) & UART01x_FR_TXFF) {
786 * No space in the FIFO, so enable the transmit interrupt
787 * so we know when there is space. Note that once we've
788 * loaded the character, we should just re-enable DMA.
793 pl011_write(uap->port.x_char, uap, REG_DR);
794 uap->port.icount.tx++;
795 uap->port.x_char = 0;
797 /* Success - restore the DMA state */
799 pl011_write(dmacr, uap, REG_DMACR);
805 * Flush the transmit buffer.
806 * Locking: called with port lock held and IRQs disabled.
808 static void pl011_dma_flush_buffer(struct uart_port *port)
809 __releases(&uap->port.lock)
810 __acquires(&uap->port.lock)
812 struct uart_amba_port *uap =
813 container_of(port, struct uart_amba_port, port);
815 if (!uap->using_tx_dma)
818 dmaengine_terminate_async(uap->dmatx.chan);
820 if (uap->dmatx.queued) {
821 dma_unmap_sg(uap->dmatx.chan->device->dev, &uap->dmatx.sg, 1,
823 uap->dmatx.queued = false;
824 uap->dmacr &= ~UART011_TXDMAE;
825 pl011_write(uap->dmacr, uap, REG_DMACR);
829 static void pl011_dma_rx_callback(void *data);
831 static int pl011_dma_rx_trigger_dma(struct uart_amba_port *uap)
833 struct dma_chan *rxchan = uap->dmarx.chan;
834 struct pl011_dmarx_data *dmarx = &uap->dmarx;
835 struct dma_async_tx_descriptor *desc;
836 struct pl011_sgbuf *sgbuf;
841 /* Start the RX DMA job */
842 sgbuf = uap->dmarx.use_buf_b ?
843 &uap->dmarx.sgbuf_b : &uap->dmarx.sgbuf_a;
844 desc = dmaengine_prep_slave_sg(rxchan, &sgbuf->sg, 1,
846 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
848 * If the DMA engine is busy and cannot prepare a
849 * channel, no big deal, the driver will fall back
850 * to interrupt mode as a result of this error code.
853 uap->dmarx.running = false;
854 dmaengine_terminate_all(rxchan);
858 /* Some data to go along to the callback */
859 desc->callback = pl011_dma_rx_callback;
860 desc->callback_param = uap;
861 dmarx->cookie = dmaengine_submit(desc);
862 dma_async_issue_pending(rxchan);
864 uap->dmacr |= UART011_RXDMAE;
865 pl011_write(uap->dmacr, uap, REG_DMACR);
866 uap->dmarx.running = true;
868 uap->im &= ~UART011_RXIM;
869 pl011_write(uap->im, uap, REG_IMSC);
875 * This is called when either the DMA job is complete, or
876 * the FIFO timeout interrupt occurred. This must be called
877 * with the port spinlock uap->port.lock held.
879 static void pl011_dma_rx_chars(struct uart_amba_port *uap,
880 u32 pending, bool use_buf_b,
883 struct tty_port *port = &uap->port.state->port;
884 struct pl011_sgbuf *sgbuf = use_buf_b ?
885 &uap->dmarx.sgbuf_b : &uap->dmarx.sgbuf_a;
887 u32 fifotaken = 0; /* only used for vdbg() */
889 struct pl011_dmarx_data *dmarx = &uap->dmarx;
892 if (uap->dmarx.poll_rate) {
893 /* The data can be taken by polling */
894 dmataken = sgbuf->sg.length - dmarx->last_residue;
895 /* Recalculate the pending size */
896 if (pending >= dmataken)
900 /* Pick the remain data from the DMA */
904 * First take all chars in the DMA pipe, then look in the FIFO.
905 * Note that tty_insert_flip_buf() tries to take as many chars
908 dma_count = tty_insert_flip_string(port, sgbuf->buf + dmataken,
911 uap->port.icount.rx += dma_count;
912 if (dma_count < pending)
913 dev_warn(uap->port.dev,
914 "couldn't insert all characters (TTY is full?)\n");
917 /* Reset the last_residue for Rx DMA poll */
918 if (uap->dmarx.poll_rate)
919 dmarx->last_residue = sgbuf->sg.length;
922 * Only continue with trying to read the FIFO if all DMA chars have
925 if (dma_count == pending && readfifo) {
926 /* Clear any error flags */
927 pl011_write(UART011_OEIS | UART011_BEIS | UART011_PEIS |
928 UART011_FEIS, uap, REG_ICR);
931 * If we read all the DMA'd characters, and we had an
932 * incomplete buffer, that could be due to an rx error, or
933 * maybe we just timed out. Read any pending chars and check
936 * Error conditions will only occur in the FIFO, these will
937 * trigger an immediate interrupt and stop the DMA job, so we
938 * will always find the error in the FIFO, never in the DMA
941 fifotaken = pl011_fifo_to_tty(uap);
944 dev_vdbg(uap->port.dev,
945 "Took %d chars from DMA buffer and %d chars from the FIFO\n",
946 dma_count, fifotaken);
947 tty_flip_buffer_push(port);
950 static void pl011_dma_rx_irq(struct uart_amba_port *uap)
952 struct pl011_dmarx_data *dmarx = &uap->dmarx;
953 struct dma_chan *rxchan = dmarx->chan;
954 struct pl011_sgbuf *sgbuf = dmarx->use_buf_b ?
955 &dmarx->sgbuf_b : &dmarx->sgbuf_a;
957 struct dma_tx_state state;
958 enum dma_status dmastat;
961 * Pause the transfer so we can trust the current counter,
962 * do this before we pause the PL011 block, else we may
965 if (dmaengine_pause(rxchan))
966 dev_err(uap->port.dev, "unable to pause DMA transfer\n");
967 dmastat = rxchan->device->device_tx_status(rxchan,
968 dmarx->cookie, &state);
969 if (dmastat != DMA_PAUSED)
970 dev_err(uap->port.dev, "unable to pause DMA transfer\n");
972 /* Disable RX DMA - incoming data will wait in the FIFO */
973 uap->dmacr &= ~UART011_RXDMAE;
974 pl011_write(uap->dmacr, uap, REG_DMACR);
975 uap->dmarx.running = false;
977 pending = sgbuf->sg.length - state.residue;
978 BUG_ON(pending > PL011_DMA_BUFFER_SIZE);
979 /* Then we terminate the transfer - we now know our residue */
980 dmaengine_terminate_all(rxchan);
983 * This will take the chars we have so far and insert
984 * into the framework.
986 pl011_dma_rx_chars(uap, pending, dmarx->use_buf_b, true);
988 /* Switch buffer & re-trigger DMA job */
989 dmarx->use_buf_b = !dmarx->use_buf_b;
990 if (pl011_dma_rx_trigger_dma(uap)) {
991 dev_dbg(uap->port.dev, "could not retrigger RX DMA job "
992 "fall back to interrupt mode\n");
993 uap->im |= UART011_RXIM;
994 pl011_write(uap->im, uap, REG_IMSC);
998 static void pl011_dma_rx_callback(void *data)
1000 struct uart_amba_port *uap = data;
1001 struct pl011_dmarx_data *dmarx = &uap->dmarx;
1002 struct dma_chan *rxchan = dmarx->chan;
1003 bool lastbuf = dmarx->use_buf_b;
1004 struct pl011_sgbuf *sgbuf = dmarx->use_buf_b ?
1005 &dmarx->sgbuf_b : &dmarx->sgbuf_a;
1007 struct dma_tx_state state;
1011 * This completion interrupt occurs typically when the
1012 * RX buffer is totally stuffed but no timeout has yet
1013 * occurred. When that happens, we just want the RX
1014 * routine to flush out the secondary DMA buffer while
1015 * we immediately trigger the next DMA job.
1017 spin_lock_irq(&uap->port.lock);
1019 * Rx data can be taken by the UART interrupts during
1020 * the DMA irq handler. So we check the residue here.
1022 rxchan->device->device_tx_status(rxchan, dmarx->cookie, &state);
1023 pending = sgbuf->sg.length - state.residue;
1024 BUG_ON(pending > PL011_DMA_BUFFER_SIZE);
1025 /* Then we terminate the transfer - we now know our residue */
1026 dmaengine_terminate_all(rxchan);
1028 uap->dmarx.running = false;
1029 dmarx->use_buf_b = !lastbuf;
1030 ret = pl011_dma_rx_trigger_dma(uap);
1032 pl011_dma_rx_chars(uap, pending, lastbuf, false);
1033 spin_unlock_irq(&uap->port.lock);
1035 * Do this check after we picked the DMA chars so we don't
1036 * get some IRQ immediately from RX.
1039 dev_dbg(uap->port.dev, "could not retrigger RX DMA job "
1040 "fall back to interrupt mode\n");
1041 uap->im |= UART011_RXIM;
1042 pl011_write(uap->im, uap, REG_IMSC);
1047 * Stop accepting received characters, when we're shutting down or
1048 * suspending this port.
1049 * Locking: called with port lock held and IRQs disabled.
1051 static inline void pl011_dma_rx_stop(struct uart_amba_port *uap)
1053 /* FIXME. Just disable the DMA enable */
1054 uap->dmacr &= ~UART011_RXDMAE;
1055 pl011_write(uap->dmacr, uap, REG_DMACR);
1059 * Timer handler for Rx DMA polling.
1060 * Every polling, It checks the residue in the dma buffer and transfer
1061 * data to the tty. Also, last_residue is updated for the next polling.
1063 static void pl011_dma_rx_poll(struct timer_list *t)
1065 struct uart_amba_port *uap = from_timer(uap, t, dmarx.timer);
1066 struct tty_port *port = &uap->port.state->port;
1067 struct pl011_dmarx_data *dmarx = &uap->dmarx;
1068 struct dma_chan *rxchan = uap->dmarx.chan;
1069 unsigned long flags;
1070 unsigned int dmataken = 0;
1071 unsigned int size = 0;
1072 struct pl011_sgbuf *sgbuf;
1074 struct dma_tx_state state;
1076 sgbuf = dmarx->use_buf_b ? &uap->dmarx.sgbuf_b : &uap->dmarx.sgbuf_a;
1077 rxchan->device->device_tx_status(rxchan, dmarx->cookie, &state);
1078 if (likely(state.residue < dmarx->last_residue)) {
1079 dmataken = sgbuf->sg.length - dmarx->last_residue;
1080 size = dmarx->last_residue - state.residue;
1081 dma_count = tty_insert_flip_string(port, sgbuf->buf + dmataken,
1083 if (dma_count == size)
1084 dmarx->last_residue = state.residue;
1085 dmarx->last_jiffies = jiffies;
1087 tty_flip_buffer_push(port);
1090 * If no data is received in poll_timeout, the driver will fall back
1091 * to interrupt mode. We will retrigger DMA at the first interrupt.
1093 if (jiffies_to_msecs(jiffies - dmarx->last_jiffies)
1094 > uap->dmarx.poll_timeout) {
1096 spin_lock_irqsave(&uap->port.lock, flags);
1097 pl011_dma_rx_stop(uap);
1098 uap->im |= UART011_RXIM;
1099 pl011_write(uap->im, uap, REG_IMSC);
1100 spin_unlock_irqrestore(&uap->port.lock, flags);
1102 uap->dmarx.running = false;
1103 dmaengine_terminate_all(rxchan);
1104 del_timer(&uap->dmarx.timer);
1106 mod_timer(&uap->dmarx.timer,
1107 jiffies + msecs_to_jiffies(uap->dmarx.poll_rate));
1111 static void pl011_dma_startup(struct uart_amba_port *uap)
1115 if (!uap->dma_probed)
1116 pl011_dma_probe(uap);
1118 if (!uap->dmatx.chan)
1121 uap->dmatx.buf = kmalloc(PL011_DMA_BUFFER_SIZE, GFP_KERNEL | __GFP_DMA);
1122 if (!uap->dmatx.buf) {
1123 dev_err(uap->port.dev, "no memory for DMA TX buffer\n");
1124 uap->port.fifosize = uap->fifosize;
1128 sg_init_one(&uap->dmatx.sg, uap->dmatx.buf, PL011_DMA_BUFFER_SIZE);
1130 /* The DMA buffer is now the FIFO the TTY subsystem can use */
1131 uap->port.fifosize = PL011_DMA_BUFFER_SIZE;
1132 uap->using_tx_dma = true;
1134 if (!uap->dmarx.chan)
1137 /* Allocate and map DMA RX buffers */
1138 ret = pl011_sgbuf_init(uap->dmarx.chan, &uap->dmarx.sgbuf_a,
1141 dev_err(uap->port.dev, "failed to init DMA %s: %d\n",
1142 "RX buffer A", ret);
1146 ret = pl011_sgbuf_init(uap->dmarx.chan, &uap->dmarx.sgbuf_b,
1149 dev_err(uap->port.dev, "failed to init DMA %s: %d\n",
1150 "RX buffer B", ret);
1151 pl011_sgbuf_free(uap->dmarx.chan, &uap->dmarx.sgbuf_a,
1156 uap->using_rx_dma = true;
1159 /* Turn on DMA error (RX/TX will be enabled on demand) */
1160 uap->dmacr |= UART011_DMAONERR;
1161 pl011_write(uap->dmacr, uap, REG_DMACR);
1164 * ST Micro variants has some specific dma burst threshold
1165 * compensation. Set this to 16 bytes, so burst will only
1166 * be issued above/below 16 bytes.
1168 if (uap->vendor->dma_threshold)
1169 pl011_write(ST_UART011_DMAWM_RX_16 | ST_UART011_DMAWM_TX_16,
1172 if (uap->using_rx_dma) {
1173 if (pl011_dma_rx_trigger_dma(uap))
1174 dev_dbg(uap->port.dev, "could not trigger initial "
1175 "RX DMA job, fall back to interrupt mode\n");
1176 if (uap->dmarx.poll_rate) {
1177 timer_setup(&uap->dmarx.timer, pl011_dma_rx_poll, 0);
1178 mod_timer(&uap->dmarx.timer,
1180 msecs_to_jiffies(uap->dmarx.poll_rate));
1181 uap->dmarx.last_residue = PL011_DMA_BUFFER_SIZE;
1182 uap->dmarx.last_jiffies = jiffies;
1187 static void pl011_dma_shutdown(struct uart_amba_port *uap)
1189 if (!(uap->using_tx_dma || uap->using_rx_dma))
1192 /* Disable RX and TX DMA */
1193 while (pl011_read(uap, REG_FR) & uap->vendor->fr_busy)
1196 spin_lock_irq(&uap->port.lock);
1197 uap->dmacr &= ~(UART011_DMAONERR | UART011_RXDMAE | UART011_TXDMAE);
1198 pl011_write(uap->dmacr, uap, REG_DMACR);
1199 spin_unlock_irq(&uap->port.lock);
1201 if (uap->using_tx_dma) {
1202 /* In theory, this should already be done by pl011_dma_flush_buffer */
1203 dmaengine_terminate_all(uap->dmatx.chan);
1204 if (uap->dmatx.queued) {
1205 dma_unmap_sg(uap->dmatx.chan->device->dev, &uap->dmatx.sg, 1,
1207 uap->dmatx.queued = false;
1210 kfree(uap->dmatx.buf);
1211 uap->using_tx_dma = false;
1214 if (uap->using_rx_dma) {
1215 dmaengine_terminate_all(uap->dmarx.chan);
1216 /* Clean up the RX DMA */
1217 pl011_sgbuf_free(uap->dmarx.chan, &uap->dmarx.sgbuf_a, DMA_FROM_DEVICE);
1218 pl011_sgbuf_free(uap->dmarx.chan, &uap->dmarx.sgbuf_b, DMA_FROM_DEVICE);
1219 if (uap->dmarx.poll_rate)
1220 del_timer_sync(&uap->dmarx.timer);
1221 uap->using_rx_dma = false;
1225 static inline bool pl011_dma_rx_available(struct uart_amba_port *uap)
1227 return uap->using_rx_dma;
1230 static inline bool pl011_dma_rx_running(struct uart_amba_port *uap)
1232 return uap->using_rx_dma && uap->dmarx.running;
1236 /* Blank functions if the DMA engine is not available */
1237 static inline void pl011_dma_remove(struct uart_amba_port *uap)
1241 static inline void pl011_dma_startup(struct uart_amba_port *uap)
1245 static inline void pl011_dma_shutdown(struct uart_amba_port *uap)
1249 static inline bool pl011_dma_tx_irq(struct uart_amba_port *uap)
1254 static inline void pl011_dma_tx_stop(struct uart_amba_port *uap)
1258 static inline bool pl011_dma_tx_start(struct uart_amba_port *uap)
1263 static inline void pl011_dma_rx_irq(struct uart_amba_port *uap)
1267 static inline void pl011_dma_rx_stop(struct uart_amba_port *uap)
1271 static inline int pl011_dma_rx_trigger_dma(struct uart_amba_port *uap)
1276 static inline bool pl011_dma_rx_available(struct uart_amba_port *uap)
1281 static inline bool pl011_dma_rx_running(struct uart_amba_port *uap)
1286 #define pl011_dma_flush_buffer NULL
1289 static void pl011_rs485_tx_stop(struct uart_amba_port *uap)
1292 * To be on the safe side only time out after twice as many iterations
1295 const int MAX_TX_DRAIN_ITERS = uap->port.fifosize * 2;
1296 struct uart_port *port = &uap->port;
1300 /* Wait until hardware tx queue is empty */
1301 while (!pl011_tx_empty(port)) {
1302 if (i > MAX_TX_DRAIN_ITERS) {
1304 "timeout while draining hardware tx queue\n");
1308 udelay(uap->rs485_tx_drain_interval);
1312 if (port->rs485.delay_rts_after_send)
1313 mdelay(port->rs485.delay_rts_after_send);
1315 cr = pl011_read(uap, REG_CR);
1317 if (port->rs485.flags & SER_RS485_RTS_AFTER_SEND)
1318 cr &= ~UART011_CR_RTS;
1320 cr |= UART011_CR_RTS;
1322 /* Disable the transmitter and reenable the transceiver */
1323 cr &= ~UART011_CR_TXE;
1324 cr |= UART011_CR_RXE;
1325 pl011_write(cr, uap, REG_CR);
1327 uap->rs485_tx_started = false;
1330 static void pl011_stop_tx(struct uart_port *port)
1332 struct uart_amba_port *uap =
1333 container_of(port, struct uart_amba_port, port);
1335 uap->im &= ~UART011_TXIM;
1336 pl011_write(uap->im, uap, REG_IMSC);
1337 pl011_dma_tx_stop(uap);
1339 if ((port->rs485.flags & SER_RS485_ENABLED) && uap->rs485_tx_started)
1340 pl011_rs485_tx_stop(uap);
1343 static bool pl011_tx_chars(struct uart_amba_port *uap, bool from_irq);
1345 /* Start TX with programmed I/O only (no DMA) */
1346 static void pl011_start_tx_pio(struct uart_amba_port *uap)
1348 if (pl011_tx_chars(uap, false)) {
1349 uap->im |= UART011_TXIM;
1350 pl011_write(uap->im, uap, REG_IMSC);
1354 static void pl011_start_tx(struct uart_port *port)
1356 struct uart_amba_port *uap =
1357 container_of(port, struct uart_amba_port, port);
1359 if (!pl011_dma_tx_start(uap))
1360 pl011_start_tx_pio(uap);
1363 static void pl011_stop_rx(struct uart_port *port)
1365 struct uart_amba_port *uap =
1366 container_of(port, struct uart_amba_port, port);
1368 uap->im &= ~(UART011_RXIM|UART011_RTIM|UART011_FEIM|
1369 UART011_PEIM|UART011_BEIM|UART011_OEIM);
1370 pl011_write(uap->im, uap, REG_IMSC);
1372 pl011_dma_rx_stop(uap);
1375 static void pl011_throttle_rx(struct uart_port *port)
1377 unsigned long flags;
1379 spin_lock_irqsave(&port->lock, flags);
1380 pl011_stop_rx(port);
1381 spin_unlock_irqrestore(&port->lock, flags);
1384 static void pl011_enable_ms(struct uart_port *port)
1386 struct uart_amba_port *uap =
1387 container_of(port, struct uart_amba_port, port);
1389 uap->im |= UART011_RIMIM|UART011_CTSMIM|UART011_DCDMIM|UART011_DSRMIM;
1390 pl011_write(uap->im, uap, REG_IMSC);
1393 static void pl011_rx_chars(struct uart_amba_port *uap)
1394 __releases(&uap->port.lock)
1395 __acquires(&uap->port.lock)
1397 pl011_fifo_to_tty(uap);
1399 spin_unlock(&uap->port.lock);
1400 tty_flip_buffer_push(&uap->port.state->port);
1402 * If we were temporarily out of DMA mode for a while,
1403 * attempt to switch back to DMA mode again.
1405 if (pl011_dma_rx_available(uap)) {
1406 if (pl011_dma_rx_trigger_dma(uap)) {
1407 dev_dbg(uap->port.dev, "could not trigger RX DMA job "
1408 "fall back to interrupt mode again\n");
1409 uap->im |= UART011_RXIM;
1410 pl011_write(uap->im, uap, REG_IMSC);
1412 #ifdef CONFIG_DMA_ENGINE
1413 /* Start Rx DMA poll */
1414 if (uap->dmarx.poll_rate) {
1415 uap->dmarx.last_jiffies = jiffies;
1416 uap->dmarx.last_residue = PL011_DMA_BUFFER_SIZE;
1417 mod_timer(&uap->dmarx.timer,
1419 msecs_to_jiffies(uap->dmarx.poll_rate));
1424 spin_lock(&uap->port.lock);
1427 static bool pl011_tx_char(struct uart_amba_port *uap, unsigned char c,
1430 if (unlikely(!from_irq) &&
1431 pl011_read(uap, REG_FR) & UART01x_FR_TXFF)
1432 return false; /* unable to transmit character */
1434 pl011_write(c, uap, REG_DR);
1436 uap->port.icount.tx++;
1441 static void pl011_rs485_tx_start(struct uart_amba_port *uap)
1443 struct uart_port *port = &uap->port;
1446 /* Enable transmitter */
1447 cr = pl011_read(uap, REG_CR);
1448 cr |= UART011_CR_TXE;
1450 /* Disable receiver if half-duplex */
1451 if (!(port->rs485.flags & SER_RS485_RX_DURING_TX))
1452 cr &= ~UART011_CR_RXE;
1454 if (port->rs485.flags & SER_RS485_RTS_ON_SEND)
1455 cr &= ~UART011_CR_RTS;
1457 cr |= UART011_CR_RTS;
1459 pl011_write(cr, uap, REG_CR);
1461 if (port->rs485.delay_rts_before_send)
1462 mdelay(port->rs485.delay_rts_before_send);
1464 uap->rs485_tx_started = true;
1467 /* Returns true if tx interrupts have to be (kept) enabled */
1468 static bool pl011_tx_chars(struct uart_amba_port *uap, bool from_irq)
1470 struct circ_buf *xmit = &uap->port.state->xmit;
1471 int count = uap->fifosize >> 1;
1473 if (uap->port.x_char) {
1474 if (!pl011_tx_char(uap, uap->port.x_char, from_irq))
1476 uap->port.x_char = 0;
1479 if (uart_circ_empty(xmit) || uart_tx_stopped(&uap->port)) {
1480 pl011_stop_tx(&uap->port);
1484 if ((uap->port.rs485.flags & SER_RS485_ENABLED) &&
1485 !uap->rs485_tx_started)
1486 pl011_rs485_tx_start(uap);
1488 /* If we are using DMA mode, try to send some characters. */
1489 if (pl011_dma_tx_irq(uap))
1493 if (likely(from_irq) && count-- == 0)
1496 if (likely(from_irq) && count == 0 &&
1497 pl011_read(uap, REG_FR) & UART01x_FR_TXFF)
1500 if (!pl011_tx_char(uap, xmit->buf[xmit->tail], from_irq))
1503 xmit->tail = (xmit->tail + 1) & (UART_XMIT_SIZE - 1);
1504 } while (!uart_circ_empty(xmit));
1506 if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
1507 uart_write_wakeup(&uap->port);
1509 if (uart_circ_empty(xmit)) {
1510 pl011_stop_tx(&uap->port);
1516 static void pl011_modem_status(struct uart_amba_port *uap)
1518 unsigned int status, delta;
1520 status = pl011_read(uap, REG_FR) & UART01x_FR_MODEM_ANY;
1522 delta = status ^ uap->old_status;
1523 uap->old_status = status;
1528 if (delta & UART01x_FR_DCD)
1529 uart_handle_dcd_change(&uap->port, status & UART01x_FR_DCD);
1531 if (delta & uap->vendor->fr_dsr)
1532 uap->port.icount.dsr++;
1534 if (delta & uap->vendor->fr_cts)
1535 uart_handle_cts_change(&uap->port,
1536 status & uap->vendor->fr_cts);
1538 wake_up_interruptible(&uap->port.state->port.delta_msr_wait);
1541 static void check_apply_cts_event_workaround(struct uart_amba_port *uap)
1543 if (!uap->vendor->cts_event_workaround)
1546 /* workaround to make sure that all bits are unlocked.. */
1547 pl011_write(0x00, uap, REG_ICR);
1550 * WA: introduce 26ns(1 uart clk) delay before W1C;
1551 * single apb access will incur 2 pclk(133.12Mhz) delay,
1552 * so add 2 dummy reads
1554 pl011_read(uap, REG_ICR);
1555 pl011_read(uap, REG_ICR);
1558 static irqreturn_t pl011_int(int irq, void *dev_id)
1560 struct uart_amba_port *uap = dev_id;
1561 unsigned long flags;
1562 unsigned int status, pass_counter = AMBA_ISR_PASS_LIMIT;
1565 spin_lock_irqsave(&uap->port.lock, flags);
1566 status = pl011_read(uap, REG_RIS) & uap->im;
1569 check_apply_cts_event_workaround(uap);
1571 pl011_write(status & ~(UART011_TXIS|UART011_RTIS|
1575 if (status & (UART011_RTIS|UART011_RXIS)) {
1576 if (pl011_dma_rx_running(uap))
1577 pl011_dma_rx_irq(uap);
1579 pl011_rx_chars(uap);
1581 if (status & (UART011_DSRMIS|UART011_DCDMIS|
1582 UART011_CTSMIS|UART011_RIMIS))
1583 pl011_modem_status(uap);
1584 if (status & UART011_TXIS)
1585 pl011_tx_chars(uap, true);
1587 if (pass_counter-- == 0)
1590 status = pl011_read(uap, REG_RIS) & uap->im;
1591 } while (status != 0);
1595 spin_unlock_irqrestore(&uap->port.lock, flags);
1597 return IRQ_RETVAL(handled);
1600 static unsigned int pl011_tx_empty(struct uart_port *port)
1602 struct uart_amba_port *uap =
1603 container_of(port, struct uart_amba_port, port);
1605 /* Allow feature register bits to be inverted to work around errata */
1606 unsigned int status = pl011_read(uap, REG_FR) ^ uap->vendor->inv_fr;
1608 return status & (uap->vendor->fr_busy | UART01x_FR_TXFF) ?
1612 static unsigned int pl011_get_mctrl(struct uart_port *port)
1614 struct uart_amba_port *uap =
1615 container_of(port, struct uart_amba_port, port);
1616 unsigned int result = 0;
1617 unsigned int status = pl011_read(uap, REG_FR);
1619 #define TIOCMBIT(uartbit, tiocmbit) \
1620 if (status & uartbit) \
1623 TIOCMBIT(UART01x_FR_DCD, TIOCM_CAR);
1624 TIOCMBIT(uap->vendor->fr_dsr, TIOCM_DSR);
1625 TIOCMBIT(uap->vendor->fr_cts, TIOCM_CTS);
1626 TIOCMBIT(uap->vendor->fr_ri, TIOCM_RNG);
1631 static void pl011_set_mctrl(struct uart_port *port, unsigned int mctrl)
1633 struct uart_amba_port *uap =
1634 container_of(port, struct uart_amba_port, port);
1637 cr = pl011_read(uap, REG_CR);
1639 #define TIOCMBIT(tiocmbit, uartbit) \
1640 if (mctrl & tiocmbit) \
1645 TIOCMBIT(TIOCM_RTS, UART011_CR_RTS);
1646 TIOCMBIT(TIOCM_DTR, UART011_CR_DTR);
1647 TIOCMBIT(TIOCM_OUT1, UART011_CR_OUT1);
1648 TIOCMBIT(TIOCM_OUT2, UART011_CR_OUT2);
1649 TIOCMBIT(TIOCM_LOOP, UART011_CR_LBE);
1651 if (port->status & UPSTAT_AUTORTS) {
1652 /* We need to disable auto-RTS if we want to turn RTS off */
1653 TIOCMBIT(TIOCM_RTS, UART011_CR_RTSEN);
1657 pl011_write(cr, uap, REG_CR);
1660 static void pl011_break_ctl(struct uart_port *port, int break_state)
1662 struct uart_amba_port *uap =
1663 container_of(port, struct uart_amba_port, port);
1664 unsigned long flags;
1667 spin_lock_irqsave(&uap->port.lock, flags);
1668 lcr_h = pl011_read(uap, REG_LCRH_TX);
1669 if (break_state == -1)
1670 lcr_h |= UART01x_LCRH_BRK;
1672 lcr_h &= ~UART01x_LCRH_BRK;
1673 pl011_write(lcr_h, uap, REG_LCRH_TX);
1674 spin_unlock_irqrestore(&uap->port.lock, flags);
1677 #ifdef CONFIG_CONSOLE_POLL
1679 static void pl011_quiesce_irqs(struct uart_port *port)
1681 struct uart_amba_port *uap =
1682 container_of(port, struct uart_amba_port, port);
1684 pl011_write(pl011_read(uap, REG_MIS), uap, REG_ICR);
1686 * There is no way to clear TXIM as this is "ready to transmit IRQ", so
1687 * we simply mask it. start_tx() will unmask it.
1689 * Note we can race with start_tx(), and if the race happens, the
1690 * polling user might get another interrupt just after we clear it.
1691 * But it should be OK and can happen even w/o the race, e.g.
1692 * controller immediately got some new data and raised the IRQ.
1694 * And whoever uses polling routines assumes that it manages the device
1695 * (including tx queue), so we're also fine with start_tx()'s caller
1698 pl011_write(pl011_read(uap, REG_IMSC) & ~UART011_TXIM, uap,
1702 static int pl011_get_poll_char(struct uart_port *port)
1704 struct uart_amba_port *uap =
1705 container_of(port, struct uart_amba_port, port);
1706 unsigned int status;
1709 * The caller might need IRQs lowered, e.g. if used with KDB NMI
1712 pl011_quiesce_irqs(port);
1714 status = pl011_read(uap, REG_FR);
1715 if (status & UART01x_FR_RXFE)
1716 return NO_POLL_CHAR;
1718 return pl011_read(uap, REG_DR);
1721 static void pl011_put_poll_char(struct uart_port *port,
1724 struct uart_amba_port *uap =
1725 container_of(port, struct uart_amba_port, port);
1727 while (pl011_read(uap, REG_FR) & UART01x_FR_TXFF)
1730 pl011_write(ch, uap, REG_DR);
1733 #endif /* CONFIG_CONSOLE_POLL */
1735 unsigned long pl011_clk_round(unsigned long clk)
1737 unsigned long scaler;
1740 * If increasing a clock by less than 0.1% changes it
1741 * from ..999.. to ..000.., round up.
1744 while (scaler * 100000 < clk)
1746 if ((clk + scaler - 1)/scaler % 1000 == 0)
1747 clk = (clk/scaler + 1) * scaler;
1752 static int pl011_hwinit(struct uart_port *port)
1754 struct uart_amba_port *uap =
1755 container_of(port, struct uart_amba_port, port);
1758 /* Optionaly enable pins to be muxed in and configured */
1759 pinctrl_pm_select_default_state(port->dev);
1762 * Try to enable the clock producer.
1764 retval = clk_prepare_enable(uap->clk);
1768 uap->port.uartclk = pl011_clk_round(clk_get_rate(uap->clk));
1770 /* Clear pending error and receive interrupts */
1771 pl011_write(UART011_OEIS | UART011_BEIS | UART011_PEIS |
1772 UART011_FEIS | UART011_RTIS | UART011_RXIS,
1776 * Save interrupts enable mask, and enable RX interrupts in case if
1777 * the interrupt is used for NMI entry.
1779 uap->im = pl011_read(uap, REG_IMSC);
1780 pl011_write(UART011_RTIM | UART011_RXIM, uap, REG_IMSC);
1782 if (dev_get_platdata(uap->port.dev)) {
1783 struct amba_pl011_data *plat;
1785 plat = dev_get_platdata(uap->port.dev);
1792 static bool pl011_split_lcrh(const struct uart_amba_port *uap)
1794 return pl011_reg_to_offset(uap, REG_LCRH_RX) !=
1795 pl011_reg_to_offset(uap, REG_LCRH_TX);
1798 static void pl011_write_lcr_h(struct uart_amba_port *uap, unsigned int lcr_h)
1800 pl011_write(lcr_h, uap, REG_LCRH_RX);
1801 if (pl011_split_lcrh(uap)) {
1804 * Wait 10 PCLKs before writing LCRH_TX register,
1805 * to get this delay write read only register 10 times
1807 for (i = 0; i < 10; ++i)
1808 pl011_write(0xff, uap, REG_MIS);
1809 pl011_write(lcr_h, uap, REG_LCRH_TX);
1813 static int pl011_allocate_irq(struct uart_amba_port *uap)
1815 pl011_write(uap->im, uap, REG_IMSC);
1817 return request_irq(uap->port.irq, pl011_int, IRQF_SHARED, "uart-pl011", uap);
1821 * Enable interrupts, only timeouts when using DMA
1822 * if initial RX DMA job failed, start in interrupt mode
1825 static void pl011_enable_interrupts(struct uart_amba_port *uap)
1827 unsigned long flags;
1830 spin_lock_irqsave(&uap->port.lock, flags);
1832 /* Clear out any spuriously appearing RX interrupts */
1833 pl011_write(UART011_RTIS | UART011_RXIS, uap, REG_ICR);
1836 * RXIS is asserted only when the RX FIFO transitions from below
1837 * to above the trigger threshold. If the RX FIFO is already
1838 * full to the threshold this can't happen and RXIS will now be
1839 * stuck off. Drain the RX FIFO explicitly to fix this:
1841 for (i = 0; i < uap->fifosize * 2; ++i) {
1842 if (pl011_read(uap, REG_FR) & UART01x_FR_RXFE)
1845 pl011_read(uap, REG_DR);
1848 uap->im = UART011_RTIM;
1849 if (!pl011_dma_rx_running(uap))
1850 uap->im |= UART011_RXIM;
1851 pl011_write(uap->im, uap, REG_IMSC);
1852 spin_unlock_irqrestore(&uap->port.lock, flags);
1855 static void pl011_unthrottle_rx(struct uart_port *port)
1857 struct uart_amba_port *uap = container_of(port, struct uart_amba_port, port);
1859 pl011_enable_interrupts(uap);
1862 static int pl011_startup(struct uart_port *port)
1864 struct uart_amba_port *uap =
1865 container_of(port, struct uart_amba_port, port);
1869 retval = pl011_hwinit(port);
1873 retval = pl011_allocate_irq(uap);
1877 pl011_write(uap->vendor->ifls, uap, REG_IFLS);
1879 spin_lock_irq(&uap->port.lock);
1881 /* restore RTS and DTR */
1882 cr = uap->old_cr & (UART011_CR_RTS | UART011_CR_DTR);
1883 cr |= UART01x_CR_UARTEN | UART011_CR_RXE;
1885 if (!(port->rs485.flags & SER_RS485_ENABLED))
1886 cr |= UART011_CR_TXE;
1888 pl011_write(cr, uap, REG_CR);
1890 spin_unlock_irq(&uap->port.lock);
1893 * initialise the old status of the modem signals
1895 uap->old_status = pl011_read(uap, REG_FR) & UART01x_FR_MODEM_ANY;
1898 pl011_dma_startup(uap);
1900 pl011_enable_interrupts(uap);
1905 clk_disable_unprepare(uap->clk);
1909 static int sbsa_uart_startup(struct uart_port *port)
1911 struct uart_amba_port *uap =
1912 container_of(port, struct uart_amba_port, port);
1915 retval = pl011_hwinit(port);
1919 retval = pl011_allocate_irq(uap);
1923 /* The SBSA UART does not support any modem status lines. */
1924 uap->old_status = 0;
1926 pl011_enable_interrupts(uap);
1931 static void pl011_shutdown_channel(struct uart_amba_port *uap,
1936 val = pl011_read(uap, lcrh);
1937 val &= ~(UART01x_LCRH_BRK | UART01x_LCRH_FEN);
1938 pl011_write(val, uap, lcrh);
1942 * disable the port. It should not disable RTS and DTR.
1943 * Also RTS and DTR state should be preserved to restore
1944 * it during startup().
1946 static void pl011_disable_uart(struct uart_amba_port *uap)
1950 uap->port.status &= ~(UPSTAT_AUTOCTS | UPSTAT_AUTORTS);
1951 spin_lock_irq(&uap->port.lock);
1952 cr = pl011_read(uap, REG_CR);
1954 cr &= UART011_CR_RTS | UART011_CR_DTR;
1955 cr |= UART01x_CR_UARTEN | UART011_CR_TXE;
1956 pl011_write(cr, uap, REG_CR);
1957 spin_unlock_irq(&uap->port.lock);
1960 * disable break condition and fifos
1962 pl011_shutdown_channel(uap, REG_LCRH_RX);
1963 if (pl011_split_lcrh(uap))
1964 pl011_shutdown_channel(uap, REG_LCRH_TX);
1967 static void pl011_disable_interrupts(struct uart_amba_port *uap)
1969 spin_lock_irq(&uap->port.lock);
1971 /* mask all interrupts and clear all pending ones */
1973 pl011_write(uap->im, uap, REG_IMSC);
1974 pl011_write(0xffff, uap, REG_ICR);
1976 spin_unlock_irq(&uap->port.lock);
1979 static void pl011_shutdown(struct uart_port *port)
1981 struct uart_amba_port *uap =
1982 container_of(port, struct uart_amba_port, port);
1984 pl011_disable_interrupts(uap);
1986 pl011_dma_shutdown(uap);
1988 if ((port->rs485.flags & SER_RS485_ENABLED) && uap->rs485_tx_started)
1989 pl011_rs485_tx_stop(uap);
1991 free_irq(uap->port.irq, uap);
1993 pl011_disable_uart(uap);
1996 * Shut down the clock producer
1998 clk_disable_unprepare(uap->clk);
1999 /* Optionally let pins go into sleep states */
2000 pinctrl_pm_select_sleep_state(port->dev);
2002 if (dev_get_platdata(uap->port.dev)) {
2003 struct amba_pl011_data *plat;
2005 plat = dev_get_platdata(uap->port.dev);
2010 if (uap->port.ops->flush_buffer)
2011 uap->port.ops->flush_buffer(port);
2014 static void sbsa_uart_shutdown(struct uart_port *port)
2016 struct uart_amba_port *uap =
2017 container_of(port, struct uart_amba_port, port);
2019 pl011_disable_interrupts(uap);
2021 free_irq(uap->port.irq, uap);
2023 if (uap->port.ops->flush_buffer)
2024 uap->port.ops->flush_buffer(port);
2028 pl011_setup_status_masks(struct uart_port *port, struct ktermios *termios)
2030 port->read_status_mask = UART011_DR_OE | 255;
2031 if (termios->c_iflag & INPCK)
2032 port->read_status_mask |= UART011_DR_FE | UART011_DR_PE;
2033 if (termios->c_iflag & (IGNBRK | BRKINT | PARMRK))
2034 port->read_status_mask |= UART011_DR_BE;
2037 * Characters to ignore
2039 port->ignore_status_mask = 0;
2040 if (termios->c_iflag & IGNPAR)
2041 port->ignore_status_mask |= UART011_DR_FE | UART011_DR_PE;
2042 if (termios->c_iflag & IGNBRK) {
2043 port->ignore_status_mask |= UART011_DR_BE;
2045 * If we're ignoring parity and break indicators,
2046 * ignore overruns too (for real raw support).
2048 if (termios->c_iflag & IGNPAR)
2049 port->ignore_status_mask |= UART011_DR_OE;
2053 * Ignore all characters if CREAD is not set.
2055 if ((termios->c_cflag & CREAD) == 0)
2056 port->ignore_status_mask |= UART_DUMMY_DR_RX;
2060 pl011_set_termios(struct uart_port *port, struct ktermios *termios,
2061 struct ktermios *old)
2063 struct uart_amba_port *uap =
2064 container_of(port, struct uart_amba_port, port);
2065 unsigned int lcr_h, old_cr;
2066 unsigned long flags;
2067 unsigned int baud, quot, clkdiv;
2070 if (uap->vendor->oversampling)
2076 * Ask the core to calculate the divisor for us.
2078 baud = uart_get_baud_rate(port, termios, old, 0,
2079 port->uartclk / clkdiv);
2080 #ifdef CONFIG_DMA_ENGINE
2082 * Adjust RX DMA polling rate with baud rate if not specified.
2084 if (uap->dmarx.auto_poll_rate)
2085 uap->dmarx.poll_rate = DIV_ROUND_UP(10000000, baud);
2088 if (baud > port->uartclk/16)
2089 quot = DIV_ROUND_CLOSEST(port->uartclk * 8, baud);
2091 quot = DIV_ROUND_CLOSEST(port->uartclk * 4, baud);
2093 switch (termios->c_cflag & CSIZE) {
2095 lcr_h = UART01x_LCRH_WLEN_5;
2098 lcr_h = UART01x_LCRH_WLEN_6;
2101 lcr_h = UART01x_LCRH_WLEN_7;
2104 lcr_h = UART01x_LCRH_WLEN_8;
2107 if (termios->c_cflag & CSTOPB)
2108 lcr_h |= UART01x_LCRH_STP2;
2109 if (termios->c_cflag & PARENB) {
2110 lcr_h |= UART01x_LCRH_PEN;
2111 if (!(termios->c_cflag & PARODD))
2112 lcr_h |= UART01x_LCRH_EPS;
2113 if (termios->c_cflag & CMSPAR)
2114 lcr_h |= UART011_LCRH_SPS;
2116 if (uap->fifosize > 1)
2117 lcr_h |= UART01x_LCRH_FEN;
2119 bits = tty_get_frame_size(termios->c_cflag);
2121 spin_lock_irqsave(&port->lock, flags);
2124 * Update the per-port timeout.
2126 uart_update_timeout(port, termios->c_cflag, baud);
2129 * Calculate the approximated time it takes to transmit one character
2130 * with the given baud rate. We use this as the poll interval when we
2131 * wait for the tx queue to empty.
2133 uap->rs485_tx_drain_interval = DIV_ROUND_UP(bits * 1000 * 1000, baud);
2135 pl011_setup_status_masks(port, termios);
2137 if (UART_ENABLE_MS(port, termios->c_cflag))
2138 pl011_enable_ms(port);
2140 if (port->rs485.flags & SER_RS485_ENABLED)
2141 termios->c_cflag &= ~CRTSCTS;
2143 old_cr = pl011_read(uap, REG_CR);
2145 if (termios->c_cflag & CRTSCTS) {
2146 if (old_cr & UART011_CR_RTS)
2147 old_cr |= UART011_CR_RTSEN;
2149 old_cr |= UART011_CR_CTSEN;
2150 port->status |= UPSTAT_AUTOCTS | UPSTAT_AUTORTS;
2152 old_cr &= ~(UART011_CR_CTSEN | UART011_CR_RTSEN);
2153 port->status &= ~(UPSTAT_AUTOCTS | UPSTAT_AUTORTS);
2156 if (uap->vendor->oversampling) {
2157 if (baud > port->uartclk / 16)
2158 old_cr |= ST_UART011_CR_OVSFACT;
2160 old_cr &= ~ST_UART011_CR_OVSFACT;
2164 * Workaround for the ST Micro oversampling variants to
2165 * increase the bitrate slightly, by lowering the divisor,
2166 * to avoid delayed sampling of start bit at high speeds,
2167 * else we see data corruption.
2169 if (uap->vendor->oversampling) {
2170 if ((baud >= 3000000) && (baud < 3250000) && (quot > 1))
2172 else if ((baud > 3250000) && (quot > 2))
2176 pl011_write(quot & 0x3f, uap, REG_FBRD);
2177 pl011_write(quot >> 6, uap, REG_IBRD);
2180 * ----------v----------v----------v----------v-----
2181 * NOTE: REG_LCRH_TX and REG_LCRH_RX MUST BE WRITTEN AFTER
2182 * REG_FBRD & REG_IBRD.
2183 * ----------^----------^----------^----------^-----
2185 pl011_write_lcr_h(uap, lcr_h);
2186 pl011_write(old_cr, uap, REG_CR);
2188 spin_unlock_irqrestore(&port->lock, flags);
2192 sbsa_uart_set_termios(struct uart_port *port, struct ktermios *termios,
2193 struct ktermios *old)
2195 struct uart_amba_port *uap =
2196 container_of(port, struct uart_amba_port, port);
2197 unsigned long flags;
2199 tty_termios_encode_baud_rate(termios, uap->fixed_baud, uap->fixed_baud);
2201 /* The SBSA UART only supports 8n1 without hardware flow control. */
2202 termios->c_cflag &= ~(CSIZE | CSTOPB | PARENB | PARODD);
2203 termios->c_cflag &= ~(CMSPAR | CRTSCTS);
2204 termios->c_cflag |= CS8 | CLOCAL;
2206 spin_lock_irqsave(&port->lock, flags);
2207 uart_update_timeout(port, CS8, uap->fixed_baud);
2208 pl011_setup_status_masks(port, termios);
2209 spin_unlock_irqrestore(&port->lock, flags);
2212 static const char *pl011_type(struct uart_port *port)
2214 struct uart_amba_port *uap =
2215 container_of(port, struct uart_amba_port, port);
2216 return uap->port.type == PORT_AMBA ? uap->type : NULL;
2220 * Configure/autoconfigure the port.
2222 static void pl011_config_port(struct uart_port *port, int flags)
2224 if (flags & UART_CONFIG_TYPE)
2225 port->type = PORT_AMBA;
2229 * verify the new serial_struct (for TIOCSSERIAL).
2231 static int pl011_verify_port(struct uart_port *port, struct serial_struct *ser)
2234 if (ser->type != PORT_UNKNOWN && ser->type != PORT_AMBA)
2236 if (ser->irq < 0 || ser->irq >= nr_irqs)
2238 if (ser->baud_base < 9600)
2240 if (port->mapbase != (unsigned long) ser->iomem_base)
2245 static int pl011_rs485_config(struct uart_port *port,
2246 struct serial_rs485 *rs485)
2248 struct uart_amba_port *uap =
2249 container_of(port, struct uart_amba_port, port);
2251 /* pick sane settings if the user hasn't */
2252 if (!(rs485->flags & SER_RS485_RTS_ON_SEND) ==
2253 !(rs485->flags & SER_RS485_RTS_AFTER_SEND)) {
2254 rs485->flags |= SER_RS485_RTS_ON_SEND;
2255 rs485->flags &= ~SER_RS485_RTS_AFTER_SEND;
2257 /* clamp the delays to [0, 100ms] */
2258 rs485->delay_rts_before_send = min(rs485->delay_rts_before_send, 100U);
2259 rs485->delay_rts_after_send = min(rs485->delay_rts_after_send, 100U);
2260 memset(rs485->padding, 0, sizeof(rs485->padding));
2262 if (port->rs485.flags & SER_RS485_ENABLED)
2263 pl011_rs485_tx_stop(uap);
2265 /* Set new configuration */
2266 port->rs485 = *rs485;
2268 /* Make sure auto RTS is disabled */
2269 if (port->rs485.flags & SER_RS485_ENABLED) {
2270 u32 cr = pl011_read(uap, REG_CR);
2272 cr &= ~UART011_CR_RTSEN;
2273 pl011_write(cr, uap, REG_CR);
2274 port->status &= ~UPSTAT_AUTORTS;
2280 static const struct uart_ops amba_pl011_pops = {
2281 .tx_empty = pl011_tx_empty,
2282 .set_mctrl = pl011_set_mctrl,
2283 .get_mctrl = pl011_get_mctrl,
2284 .stop_tx = pl011_stop_tx,
2285 .start_tx = pl011_start_tx,
2286 .stop_rx = pl011_stop_rx,
2287 .throttle = pl011_throttle_rx,
2288 .unthrottle = pl011_unthrottle_rx,
2289 .enable_ms = pl011_enable_ms,
2290 .break_ctl = pl011_break_ctl,
2291 .startup = pl011_startup,
2292 .shutdown = pl011_shutdown,
2293 .flush_buffer = pl011_dma_flush_buffer,
2294 .set_termios = pl011_set_termios,
2296 .config_port = pl011_config_port,
2297 .verify_port = pl011_verify_port,
2298 #ifdef CONFIG_CONSOLE_POLL
2299 .poll_init = pl011_hwinit,
2300 .poll_get_char = pl011_get_poll_char,
2301 .poll_put_char = pl011_put_poll_char,
2305 static void sbsa_uart_set_mctrl(struct uart_port *port, unsigned int mctrl)
2309 static unsigned int sbsa_uart_get_mctrl(struct uart_port *port)
2314 static const struct uart_ops sbsa_uart_pops = {
2315 .tx_empty = pl011_tx_empty,
2316 .set_mctrl = sbsa_uart_set_mctrl,
2317 .get_mctrl = sbsa_uart_get_mctrl,
2318 .stop_tx = pl011_stop_tx,
2319 .start_tx = pl011_start_tx,
2320 .stop_rx = pl011_stop_rx,
2321 .startup = sbsa_uart_startup,
2322 .shutdown = sbsa_uart_shutdown,
2323 .set_termios = sbsa_uart_set_termios,
2325 .config_port = pl011_config_port,
2326 .verify_port = pl011_verify_port,
2327 #ifdef CONFIG_CONSOLE_POLL
2328 .poll_init = pl011_hwinit,
2329 .poll_get_char = pl011_get_poll_char,
2330 .poll_put_char = pl011_put_poll_char,
2334 static struct uart_amba_port *amba_ports[UART_NR];
2336 #ifdef CONFIG_SERIAL_AMBA_PL011_CONSOLE
2338 static void pl011_console_putchar(struct uart_port *port, int ch)
2340 struct uart_amba_port *uap =
2341 container_of(port, struct uart_amba_port, port);
2343 while (pl011_read(uap, REG_FR) & UART01x_FR_TXFF)
2345 pl011_write(ch, uap, REG_DR);
2349 pl011_console_write(struct console *co, const char *s, unsigned int count)
2351 struct uart_amba_port *uap = amba_ports[co->index];
2352 unsigned int old_cr = 0, new_cr;
2353 unsigned long flags;
2356 clk_enable(uap->clk);
2358 local_irq_save(flags);
2359 if (uap->port.sysrq)
2361 else if (oops_in_progress)
2362 locked = spin_trylock(&uap->port.lock);
2364 spin_lock(&uap->port.lock);
2367 * First save the CR then disable the interrupts
2369 if (!uap->vendor->always_enabled) {
2370 old_cr = pl011_read(uap, REG_CR);
2371 new_cr = old_cr & ~UART011_CR_CTSEN;
2372 new_cr |= UART01x_CR_UARTEN | UART011_CR_TXE;
2373 pl011_write(new_cr, uap, REG_CR);
2376 uart_console_write(&uap->port, s, count, pl011_console_putchar);
2379 * Finally, wait for transmitter to become empty and restore the
2380 * TCR. Allow feature register bits to be inverted to work around
2383 while ((pl011_read(uap, REG_FR) ^ uap->vendor->inv_fr)
2384 & uap->vendor->fr_busy)
2386 if (!uap->vendor->always_enabled)
2387 pl011_write(old_cr, uap, REG_CR);
2390 spin_unlock(&uap->port.lock);
2391 local_irq_restore(flags);
2393 clk_disable(uap->clk);
2396 static void pl011_console_get_options(struct uart_amba_port *uap, int *baud,
2397 int *parity, int *bits)
2399 if (pl011_read(uap, REG_CR) & UART01x_CR_UARTEN) {
2400 unsigned int lcr_h, ibrd, fbrd;
2402 lcr_h = pl011_read(uap, REG_LCRH_TX);
2405 if (lcr_h & UART01x_LCRH_PEN) {
2406 if (lcr_h & UART01x_LCRH_EPS)
2412 if ((lcr_h & 0x60) == UART01x_LCRH_WLEN_7)
2417 ibrd = pl011_read(uap, REG_IBRD);
2418 fbrd = pl011_read(uap, REG_FBRD);
2420 *baud = uap->port.uartclk * 4 / (64 * ibrd + fbrd);
2422 if (uap->vendor->oversampling) {
2423 if (pl011_read(uap, REG_CR)
2424 & ST_UART011_CR_OVSFACT)
2430 static int pl011_console_setup(struct console *co, char *options)
2432 struct uart_amba_port *uap;
2440 * Check whether an invalid uart number has been specified, and
2441 * if so, search for the first available port that does have
2444 if (co->index >= UART_NR)
2446 uap = amba_ports[co->index];
2450 /* Allow pins to be muxed in and configured */
2451 pinctrl_pm_select_default_state(uap->port.dev);
2453 ret = clk_prepare(uap->clk);
2457 if (dev_get_platdata(uap->port.dev)) {
2458 struct amba_pl011_data *plat;
2460 plat = dev_get_platdata(uap->port.dev);
2465 uap->port.uartclk = pl011_clk_round(clk_get_rate(uap->clk));
2467 if (uap->vendor->fixed_options) {
2468 baud = uap->fixed_baud;
2471 uart_parse_options(options,
2472 &baud, &parity, &bits, &flow);
2474 pl011_console_get_options(uap, &baud, &parity, &bits);
2477 return uart_set_options(&uap->port, co, baud, parity, bits, flow);
2481 * pl011_console_match - non-standard console matching
2482 * @co: registering console
2483 * @name: name from console command line
2484 * @idx: index from console command line
2485 * @options: ptr to option string from console command line
2487 * Only attempts to match console command lines of the form:
2488 * console=pl011,mmio|mmio32,<addr>[,<options>]
2489 * console=pl011,0x<addr>[,<options>]
2490 * This form is used to register an initial earlycon boot console and
2491 * replace it with the amba_console at pl011 driver init.
2493 * Performs console setup for a match (as required by interface)
2494 * If no <options> are specified, then assume the h/w is already setup.
2496 * Returns 0 if console matches; otherwise non-zero to use default matching
2498 static int pl011_console_match(struct console *co, char *name, int idx,
2501 unsigned char iotype;
2502 resource_size_t addr;
2506 * Systems affected by the Qualcomm Technologies QDF2400 E44 erratum
2507 * have a distinct console name, so make sure we check for that.
2508 * The actual implementation of the erratum occurs in the probe
2511 if ((strcmp(name, "qdf2400_e44") != 0) && (strcmp(name, "pl011") != 0))
2514 if (uart_parse_earlycon(options, &iotype, &addr, &options))
2517 if (iotype != UPIO_MEM && iotype != UPIO_MEM32)
2520 /* try to match the port specified on the command line */
2521 for (i = 0; i < ARRAY_SIZE(amba_ports); i++) {
2522 struct uart_port *port;
2527 port = &amba_ports[i]->port;
2529 if (port->mapbase != addr)
2534 return pl011_console_setup(co, options);
2540 static struct uart_driver amba_reg;
2541 static struct console amba_console = {
2543 .write = pl011_console_write,
2544 .device = uart_console_device,
2545 .setup = pl011_console_setup,
2546 .match = pl011_console_match,
2547 .flags = CON_PRINTBUFFER | CON_ANYTIME,
2552 #define AMBA_CONSOLE (&amba_console)
2554 static void qdf2400_e44_putc(struct uart_port *port, int c)
2556 while (readl(port->membase + UART01x_FR) & UART01x_FR_TXFF)
2558 writel(c, port->membase + UART01x_DR);
2559 while (!(readl(port->membase + UART01x_FR) & UART011_FR_TXFE))
2563 static void qdf2400_e44_early_write(struct console *con, const char *s, unsigned n)
2565 struct earlycon_device *dev = con->data;
2567 uart_console_write(&dev->port, s, n, qdf2400_e44_putc);
2570 static void pl011_putc(struct uart_port *port, int c)
2572 while (readl(port->membase + UART01x_FR) & UART01x_FR_TXFF)
2574 if (port->iotype == UPIO_MEM32)
2575 writel(c, port->membase + UART01x_DR);
2577 writeb(c, port->membase + UART01x_DR);
2578 while (readl(port->membase + UART01x_FR) & UART01x_FR_BUSY)
2582 static void pl011_early_write(struct console *con, const char *s, unsigned n)
2584 struct earlycon_device *dev = con->data;
2586 uart_console_write(&dev->port, s, n, pl011_putc);
2589 #ifdef CONFIG_CONSOLE_POLL
2590 static int pl011_getc(struct uart_port *port)
2592 if (readl(port->membase + UART01x_FR) & UART01x_FR_RXFE)
2593 return NO_POLL_CHAR;
2595 if (port->iotype == UPIO_MEM32)
2596 return readl(port->membase + UART01x_DR);
2598 return readb(port->membase + UART01x_DR);
2601 static int pl011_early_read(struct console *con, char *s, unsigned int n)
2603 struct earlycon_device *dev = con->data;
2604 int ch, num_read = 0;
2606 while (num_read < n) {
2607 ch = pl011_getc(&dev->port);
2608 if (ch == NO_POLL_CHAR)
2617 #define pl011_early_read NULL
2621 * On non-ACPI systems, earlycon is enabled by specifying
2622 * "earlycon=pl011,<address>" on the kernel command line.
2624 * On ACPI ARM64 systems, an "early" console is enabled via the SPCR table,
2625 * by specifying only "earlycon" on the command line. Because it requires
2626 * SPCR, the console starts after ACPI is parsed, which is later than a
2627 * traditional early console.
2629 * To get the traditional early console that starts before ACPI is parsed,
2630 * specify the full "earlycon=pl011,<address>" option.
2632 static int __init pl011_early_console_setup(struct earlycon_device *device,
2635 if (!device->port.membase)
2638 device->con->write = pl011_early_write;
2639 device->con->read = pl011_early_read;
2643 OF_EARLYCON_DECLARE(pl011, "arm,pl011", pl011_early_console_setup);
2644 OF_EARLYCON_DECLARE(pl011, "arm,sbsa-uart", pl011_early_console_setup);
2647 * On Qualcomm Datacenter Technologies QDF2400 SOCs affected by
2648 * Erratum 44, traditional earlycon can be enabled by specifying
2649 * "earlycon=qdf2400_e44,<address>". Any options are ignored.
2651 * Alternatively, you can just specify "earlycon", and the early console
2652 * will be enabled with the information from the SPCR table. In this
2653 * case, the SPCR code will detect the need for the E44 work-around,
2654 * and set the console name to "qdf2400_e44".
2657 qdf2400_e44_early_console_setup(struct earlycon_device *device,
2660 if (!device->port.membase)
2663 device->con->write = qdf2400_e44_early_write;
2666 EARLYCON_DECLARE(qdf2400_e44, qdf2400_e44_early_console_setup);
2669 #define AMBA_CONSOLE NULL
2672 static struct uart_driver amba_reg = {
2673 .owner = THIS_MODULE,
2674 .driver_name = "ttyAMA",
2675 .dev_name = "ttyAMA",
2676 .major = SERIAL_AMBA_MAJOR,
2677 .minor = SERIAL_AMBA_MINOR,
2679 .cons = AMBA_CONSOLE,
2683 static int pl011_probe_dt_alias(int index, struct device *dev)
2685 struct device_node *np;
2686 static bool seen_dev_with_alias = false;
2687 static bool seen_dev_without_alias = false;
2690 if (!IS_ENABLED(CONFIG_OF))
2697 ret = of_alias_get_id(np, "serial");
2699 seen_dev_without_alias = true;
2702 seen_dev_with_alias = true;
2703 if (ret >= ARRAY_SIZE(amba_ports) || amba_ports[ret] != NULL) {
2704 dev_warn(dev, "requested serial port %d not available.\n", ret);
2709 if (seen_dev_with_alias && seen_dev_without_alias)
2710 dev_warn(dev, "aliased and non-aliased serial devices found in device tree. Serial port enumeration may be unpredictable.\n");
2716 /* unregisters the driver also if no more ports are left */
2717 static void pl011_unregister_port(struct uart_amba_port *uap)
2722 for (i = 0; i < ARRAY_SIZE(amba_ports); i++) {
2723 if (amba_ports[i] == uap)
2724 amba_ports[i] = NULL;
2725 else if (amba_ports[i])
2728 pl011_dma_remove(uap);
2730 uart_unregister_driver(&amba_reg);
2733 static int pl011_find_free_port(void)
2737 for (i = 0; i < ARRAY_SIZE(amba_ports); i++)
2738 if (amba_ports[i] == NULL)
2744 static int pl011_get_rs485_mode(struct uart_amba_port *uap)
2746 struct uart_port *port = &uap->port;
2747 struct serial_rs485 *rs485 = &port->rs485;
2750 ret = uart_get_rs485_mode(port);
2754 /* clamp the delays to [0, 100ms] */
2755 rs485->delay_rts_before_send = min(rs485->delay_rts_before_send, 100U);
2756 rs485->delay_rts_after_send = min(rs485->delay_rts_after_send, 100U);
2761 static int pl011_setup_port(struct device *dev, struct uart_amba_port *uap,
2762 struct resource *mmiobase, int index)
2767 base = devm_ioremap_resource(dev, mmiobase);
2769 return PTR_ERR(base);
2771 /* Don't use DT serial<n> aliases - it causes the device to
2772 be renumbered to ttyAMA1 if it is the second serial port in the
2773 system, even though the other one is ttyS0. The 8250 driver
2774 doesn't use this logic, so always remains ttyS0.
2775 index = pl011_probe_dt_alias(index, dev);
2779 uap->port.dev = dev;
2780 uap->port.mapbase = mmiobase->start;
2781 uap->port.membase = base;
2782 uap->port.fifosize = uap->fifosize;
2783 uap->port.has_sysrq = IS_ENABLED(CONFIG_SERIAL_AMBA_PL011_CONSOLE);
2784 uap->port.flags = UPF_BOOT_AUTOCONF;
2785 uap->port.line = index;
2787 ret = pl011_get_rs485_mode(uap);
2791 amba_ports[index] = uap;
2796 static int pl011_register_port(struct uart_amba_port *uap)
2800 /* Ensure interrupts from this UART are masked and cleared */
2801 pl011_write(0, uap, REG_IMSC);
2802 pl011_write(0xffff, uap, REG_ICR);
2804 if (!amba_reg.state) {
2805 ret = uart_register_driver(&amba_reg);
2807 dev_err(uap->port.dev,
2808 "Failed to register AMBA-PL011 driver\n");
2809 for (i = 0; i < ARRAY_SIZE(amba_ports); i++)
2810 if (amba_ports[i] == uap)
2811 amba_ports[i] = NULL;
2816 ret = uart_add_one_port(&amba_reg, &uap->port);
2818 pl011_unregister_port(uap);
2823 static int pl011_probe(struct amba_device *dev, const struct amba_id *id)
2825 struct uart_amba_port *uap;
2826 struct vendor_data *vendor = id->data;
2829 portnr = pl011_find_free_port();
2833 uap = devm_kzalloc(&dev->dev, sizeof(struct uart_amba_port),
2838 uap->clk = devm_clk_get(&dev->dev, NULL);
2839 if (IS_ERR(uap->clk))
2840 return PTR_ERR(uap->clk);
2842 if (of_property_read_bool(dev->dev.of_node, "cts-event-workaround")) {
2843 vendor->cts_event_workaround = true;
2844 dev_info(&dev->dev, "cts_event_workaround enabled\n");
2847 uap->reg_offset = vendor->reg_offset;
2848 uap->vendor = vendor;
2849 uap->fifosize = vendor->get_fifosize(dev);
2850 uap->port.iotype = vendor->access_32b ? UPIO_MEM32 : UPIO_MEM;
2851 uap->port.irq = dev->irq[0];
2852 uap->port.ops = &amba_pl011_pops;
2853 uap->port.rs485_config = pl011_rs485_config;
2854 snprintf(uap->type, sizeof(uap->type), "PL011 rev%u", amba_rev(dev));
2856 ret = pl011_setup_port(&dev->dev, uap, &dev->res, portnr);
2860 amba_set_drvdata(dev, uap);
2862 return pl011_register_port(uap);
2865 static void pl011_remove(struct amba_device *dev)
2867 struct uart_amba_port *uap = amba_get_drvdata(dev);
2869 uart_remove_one_port(&amba_reg, &uap->port);
2870 pl011_unregister_port(uap);
2873 #ifdef CONFIG_PM_SLEEP
2874 static int pl011_suspend(struct device *dev)
2876 struct uart_amba_port *uap = dev_get_drvdata(dev);
2881 return uart_suspend_port(&amba_reg, &uap->port);
2884 static int pl011_resume(struct device *dev)
2886 struct uart_amba_port *uap = dev_get_drvdata(dev);
2891 return uart_resume_port(&amba_reg, &uap->port);
2895 static SIMPLE_DEV_PM_OPS(pl011_dev_pm_ops, pl011_suspend, pl011_resume);
2897 static int sbsa_uart_probe(struct platform_device *pdev)
2899 struct uart_amba_port *uap;
2905 * Check the mandatory baud rate parameter in the DT node early
2906 * so that we can easily exit with the error.
2908 if (pdev->dev.of_node) {
2909 struct device_node *np = pdev->dev.of_node;
2911 ret = of_property_read_u32(np, "current-speed", &baudrate);
2918 portnr = pl011_find_free_port();
2922 uap = devm_kzalloc(&pdev->dev, sizeof(struct uart_amba_port),
2927 ret = platform_get_irq(pdev, 0);
2930 uap->port.irq = ret;
2932 #ifdef CONFIG_ACPI_SPCR_TABLE
2933 if (qdf2400_e44_present) {
2934 dev_info(&pdev->dev, "working around QDF2400 SoC erratum 44\n");
2935 uap->vendor = &vendor_qdt_qdf2400_e44;
2938 uap->vendor = &vendor_sbsa;
2940 uap->reg_offset = uap->vendor->reg_offset;
2942 uap->port.iotype = uap->vendor->access_32b ? UPIO_MEM32 : UPIO_MEM;
2943 uap->port.ops = &sbsa_uart_pops;
2944 uap->fixed_baud = baudrate;
2946 snprintf(uap->type, sizeof(uap->type), "SBSA");
2948 r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2950 ret = pl011_setup_port(&pdev->dev, uap, r, portnr);
2954 platform_set_drvdata(pdev, uap);
2956 return pl011_register_port(uap);
2959 static int sbsa_uart_remove(struct platform_device *pdev)
2961 struct uart_amba_port *uap = platform_get_drvdata(pdev);
2963 uart_remove_one_port(&amba_reg, &uap->port);
2964 pl011_unregister_port(uap);
2968 static const struct of_device_id sbsa_uart_of_match[] = {
2969 { .compatible = "arm,sbsa-uart", },
2972 MODULE_DEVICE_TABLE(of, sbsa_uart_of_match);
2974 static const struct acpi_device_id __maybe_unused sbsa_uart_acpi_match[] = {
2979 MODULE_DEVICE_TABLE(acpi, sbsa_uart_acpi_match);
2981 static struct platform_driver arm_sbsa_uart_platform_driver = {
2982 .probe = sbsa_uart_probe,
2983 .remove = sbsa_uart_remove,
2985 .name = "sbsa-uart",
2986 .pm = &pl011_dev_pm_ops,
2987 .of_match_table = of_match_ptr(sbsa_uart_of_match),
2988 .acpi_match_table = ACPI_PTR(sbsa_uart_acpi_match),
2989 .suppress_bind_attrs = IS_BUILTIN(CONFIG_SERIAL_AMBA_PL011),
2993 static const struct amba_id pl011_ids[] = {
2997 .data = &vendor_arm,
3005 .id = AMBA_LINUX_ID(0x00, 0x1, 0xffe),
3007 .data = &vendor_zte,
3012 MODULE_DEVICE_TABLE(amba, pl011_ids);
3014 static struct amba_driver pl011_driver = {
3016 .name = "uart-pl011",
3017 .pm = &pl011_dev_pm_ops,
3018 .suppress_bind_attrs = IS_BUILTIN(CONFIG_SERIAL_AMBA_PL011),
3020 .id_table = pl011_ids,
3021 .probe = pl011_probe,
3022 .remove = pl011_remove,
3025 static int __init pl011_init(void)
3027 printk(KERN_INFO "Serial: AMBA PL011 UART driver\n");
3029 if (platform_driver_register(&arm_sbsa_uart_platform_driver))
3030 pr_warn("could not register SBSA UART platform driver\n");
3031 return amba_driver_register(&pl011_driver);
3034 static void __exit pl011_exit(void)
3036 platform_driver_unregister(&arm_sbsa_uart_platform_driver);
3037 amba_driver_unregister(&pl011_driver);
3041 * While this can be a module, if builtin it's most likely the console
3042 * So let's leave module_exit but move module_init to an earlier place
3044 arch_initcall(pl011_init);
3045 module_exit(pl011_exit);
3047 MODULE_AUTHOR("ARM Ltd/Deep Blue Solutions Ltd");
3048 MODULE_DESCRIPTION("ARM AMBA serial port driver");
3049 MODULE_LICENSE("GPL");