drivers/tty/serial/stm32-usart.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) Maxime Coquelin 2015
   4  * Copyright (C) STMicroelectronics SA 2017
   5  * Authors:  Maxime Coquelin <mcoquelin.stm32@gmail.com>
   6  *           Gerald Baeza <gerald.baeza@foss.st.com>
   7  *           Erwan Le Ray <erwan.leray@foss.st.com>
   8  *
   9  * Inspired by st-asc.c from STMicroelectronics (c)
  10  */
  11
  12 #include <linux/clk.h>
  13 #include <linux/console.h>
  14 #include <linux/delay.h>
  15 #include <linux/dma-direction.h>
  16 #include <linux/dmaengine.h>
  17 #include <linux/dma-mapping.h>
  18 #include <linux/io.h>
  19 #include <linux/iopoll.h>
  20 #include <linux/irq.h>
  21 #include <linux/module.h>
  22 #include <linux/of.h>
  23 #include <linux/of_platform.h>
  24 #include <linux/pinctrl/consumer.h>
  25 #include <linux/platform_device.h>
  26 #include <linux/pm_runtime.h>
  27 #include <linux/pm_wakeirq.h>
  28 #include <linux/serial_core.h>
  29 #include <linux/serial.h>
  30 #include <linux/spinlock.h>
  31 #include <linux/sysrq.h>
  32 #include <linux/tty_flip.h>
  33 #include <linux/tty.h>
  34
  35 #include "serial_mctrl_gpio.h"
  36 #include "stm32-usart.h"
  37
  38
  39 /* Register offsets */
  40 static struct stm32_usart_info __maybe_unused stm32f4_info = {
  41         .ofs = {
  42                 .isr    = 0x00,
  43                 .rdr    = 0x04,
  44                 .tdr    = 0x04,
  45                 .brr    = 0x08,
  46                 .cr1    = 0x0c,
  47                 .cr2    = 0x10,
  48                 .cr3    = 0x14,
  49                 .gtpr   = 0x18,
  50                 .rtor   = UNDEF_REG,
  51                 .rqr    = UNDEF_REG,
  52                 .icr    = UNDEF_REG,
  53         },
  54         .cfg = {
  55                 .uart_enable_bit = 13,
  56                 .has_7bits_data = false,
  57                 .fifosize = 1,
  58         }
  59 };
  60
  61 static struct stm32_usart_info __maybe_unused stm32f7_info = {
  62         .ofs = {
  63                 .cr1    = 0x00,
  64                 .cr2    = 0x04,
  65                 .cr3    = 0x08,
  66                 .brr    = 0x0c,
  67                 .gtpr   = 0x10,
  68                 .rtor   = 0x14,
  69                 .rqr    = 0x18,
  70                 .isr    = 0x1c,
  71                 .icr    = 0x20,
  72                 .rdr    = 0x24,
  73                 .tdr    = 0x28,
  74         },
  75         .cfg = {
  76                 .uart_enable_bit = 0,
  77                 .has_7bits_data = true,
  78                 .has_swap = true,
  79                 .fifosize = 1,
  80         }
  81 };
  82
  83 static struct stm32_usart_info __maybe_unused stm32h7_info = {
  84         .ofs = {
  85                 .cr1    = 0x00,
  86                 .cr2    = 0x04,
  87                 .cr3    = 0x08,
  88                 .brr    = 0x0c,
  89                 .gtpr   = 0x10,
  90                 .rtor   = 0x14,
  91                 .rqr    = 0x18,
  92                 .isr    = 0x1c,
  93                 .icr    = 0x20,
  94                 .rdr    = 0x24,
  95                 .tdr    = 0x28,
  96         },
  97         .cfg = {
  98                 .uart_enable_bit = 0,
  99                 .has_7bits_data = true,
 100                 .has_swap = true,
 101                 .has_wakeup = true,
 102                 .has_fifo = true,
 103                 .fifosize = 16,
 104         }
 105 };
 106
 107 static void stm32_usart_stop_tx(struct uart_port *port);
 108 static void stm32_usart_transmit_chars(struct uart_port *port);
 109 static void __maybe_unused stm32_usart_console_putchar(struct uart_port *port, unsigned char ch);
 110
 111 static inline struct stm32_port *to_stm32_port(struct uart_port *port)
 112 {
 113         return container_of(port, struct stm32_port, port);
 114 }
 115
 116 static void stm32_usart_set_bits(struct uart_port *port, u32 reg, u32 bits)
 117 {
 118         u32 val;
 119
 120         val = readl_relaxed(port->membase + reg);
 121         val |= bits;
 122         writel_relaxed(val, port->membase + reg);
 123 }
 124
 125 static void stm32_usart_clr_bits(struct uart_port *port, u32 reg, u32 bits)
 126 {
 127         u32 val;
 128
 129         val = readl_relaxed(port->membase + reg);
 130         val &= ~bits;
 131         writel_relaxed(val, port->membase + reg);
 132 }
 133
 134 static unsigned int stm32_usart_tx_empty(struct uart_port *port)
 135 {
 136         struct stm32_port *stm32_port = to_stm32_port(port);
 137         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 138
 139         if (readl_relaxed(port->membase + ofs->isr) & USART_SR_TC)
 140                 return TIOCSER_TEMT;
 141
 142         return 0;
 143 }
 144
 145 static void stm32_usart_rs485_rts_enable(struct uart_port *port)
 146 {
 147         struct stm32_port *stm32_port = to_stm32_port(port);
 148         struct serial_rs485 *rs485conf = &port->rs485;
 149
 150         if (stm32_port->hw_flow_control ||
 151             !(rs485conf->flags & SER_RS485_ENABLED))
 152                 return;
 153
 154         if (rs485conf->flags & SER_RS485_RTS_ON_SEND) {
 155                 mctrl_gpio_set(stm32_port->gpios,
 156                                stm32_port->port.mctrl | TIOCM_RTS);
 157         } else {
 158                 mctrl_gpio_set(stm32_port->gpios,
 159                                stm32_port->port.mctrl & ~TIOCM_RTS);
 160         }
 161 }
 162
 163 static void stm32_usart_rs485_rts_disable(struct uart_port *port)
 164 {
 165         struct stm32_port *stm32_port = to_stm32_port(port);
 166         struct serial_rs485 *rs485conf = &port->rs485;
 167
 168         if (stm32_port->hw_flow_control ||
 169             !(rs485conf->flags & SER_RS485_ENABLED))
 170                 return;
 171
 172         if (rs485conf->flags & SER_RS485_RTS_ON_SEND) {
 173                 mctrl_gpio_set(stm32_port->gpios,
 174                                stm32_port->port.mctrl & ~TIOCM_RTS);
 175         } else {
 176                 mctrl_gpio_set(stm32_port->gpios,
 177                                stm32_port->port.mctrl | TIOCM_RTS);
 178         }
 179 }
 180
 181 static void stm32_usart_config_reg_rs485(u32 *cr1, u32 *cr3, u32 delay_ADE,
 182                                          u32 delay_DDE, u32 baud)
 183 {
 184         u32 rs485_deat_dedt;
 185         u32 rs485_deat_dedt_max = (USART_CR1_DEAT_MASK >> USART_CR1_DEAT_SHIFT);
 186         bool over8;
 187
 188         *cr3 |= USART_CR3_DEM;
 189         over8 = *cr1 & USART_CR1_OVER8;
 190
 191         *cr1 &= ~(USART_CR1_DEDT_MASK | USART_CR1_DEAT_MASK);
 192
 193         if (over8)
 194                 rs485_deat_dedt = delay_ADE * baud * 8;
 195         else
 196                 rs485_deat_dedt = delay_ADE * baud * 16;
 197
 198         rs485_deat_dedt = DIV_ROUND_CLOSEST(rs485_deat_dedt, 1000);
 199         rs485_deat_dedt = rs485_deat_dedt > rs485_deat_dedt_max ?
 200                           rs485_deat_dedt_max : rs485_deat_dedt;
 201         rs485_deat_dedt = (rs485_deat_dedt << USART_CR1_DEAT_SHIFT) &
 202                            USART_CR1_DEAT_MASK;
 203         *cr1 |= rs485_deat_dedt;
 204
 205         if (over8)
 206                 rs485_deat_dedt = delay_DDE * baud * 8;
 207         else
 208                 rs485_deat_dedt = delay_DDE * baud * 16;
 209
 210         rs485_deat_dedt = DIV_ROUND_CLOSEST(rs485_deat_dedt, 1000);
 211         rs485_deat_dedt = rs485_deat_dedt > rs485_deat_dedt_max ?
 212                           rs485_deat_dedt_max : rs485_deat_dedt;
 213         rs485_deat_dedt = (rs485_deat_dedt << USART_CR1_DEDT_SHIFT) &
 214                            USART_CR1_DEDT_MASK;
 215         *cr1 |= rs485_deat_dedt;
 216 }
 217
 218 static int stm32_usart_config_rs485(struct uart_port *port, struct ktermios *termios,
 219                                     struct serial_rs485 *rs485conf)
 220 {
 221         struct stm32_port *stm32_port = to_stm32_port(port);
 222         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 223         const struct stm32_usart_config *cfg = &stm32_port->info->cfg;
 224         u32 usartdiv, baud, cr1, cr3;
 225         bool over8;
 226
 227         stm32_usart_clr_bits(port, ofs->cr1, BIT(cfg->uart_enable_bit));
 228
 229         rs485conf->flags |= SER_RS485_RX_DURING_TX;
 230
 231         if (rs485conf->flags & SER_RS485_ENABLED) {
 232                 cr1 = readl_relaxed(port->membase + ofs->cr1);
 233                 cr3 = readl_relaxed(port->membase + ofs->cr3);
 234                 usartdiv = readl_relaxed(port->membase + ofs->brr);
 235                 usartdiv = usartdiv & GENMASK(15, 0);
 236                 over8 = cr1 & USART_CR1_OVER8;
 237
 238                 if (over8)
 239                         usartdiv = usartdiv | (usartdiv & GENMASK(4, 0))
 240                                    << USART_BRR_04_R_SHIFT;
 241
 242                 baud = DIV_ROUND_CLOSEST(port->uartclk, usartdiv);
 243                 stm32_usart_config_reg_rs485(&cr1, &cr3,
 244                                              rs485conf->delay_rts_before_send,
 245                                              rs485conf->delay_rts_after_send,
 246                                              baud);
 247
 248                 if (rs485conf->flags & SER_RS485_RTS_ON_SEND)
 249                         cr3 &= ~USART_CR3_DEP;
 250                 else
 251                         cr3 |= USART_CR3_DEP;
 252
 253                 writel_relaxed(cr3, port->membase + ofs->cr3);
 254                 writel_relaxed(cr1, port->membase + ofs->cr1);
 255         } else {
 256                 stm32_usart_clr_bits(port, ofs->cr3,
 257                                      USART_CR3_DEM | USART_CR3_DEP);
 258                 stm32_usart_clr_bits(port, ofs->cr1,
 259                                      USART_CR1_DEDT_MASK | USART_CR1_DEAT_MASK);
 260         }
 261
 262         stm32_usart_set_bits(port, ofs->cr1, BIT(cfg->uart_enable_bit));
 263
 264         /* Adjust RTS polarity in case it's driven in software */
 265         if (stm32_usart_tx_empty(port))
 266                 stm32_usart_rs485_rts_disable(port);
 267         else
 268                 stm32_usart_rs485_rts_enable(port);
 269
 270         return 0;
 271 }
 272
 273 static int stm32_usart_init_rs485(struct uart_port *port,
 274                                   struct platform_device *pdev)
 275 {
 276         struct serial_rs485 *rs485conf = &port->rs485;
 277
 278         rs485conf->flags = 0;
 279         rs485conf->delay_rts_before_send = 0;
 280         rs485conf->delay_rts_after_send = 0;
 281
 282         if (!pdev->dev.of_node)
 283                 return -ENODEV;
 284
 285         return uart_get_rs485_mode(port);
 286 }
 287
 288 static bool stm32_usart_rx_dma_enabled(struct uart_port *port)
 289 {
 290         struct stm32_port *stm32_port = to_stm32_port(port);
 291         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 292
 293         if (!stm32_port->rx_ch)
 294                 return false;
 295
 296         return !!(readl_relaxed(port->membase + ofs->cr3) & USART_CR3_DMAR);
 297 }
 298
 299 /* Return true when data is pending (in pio mode), and false when no data is pending. */
 300 static bool stm32_usart_pending_rx_pio(struct uart_port *port, u32 *sr)
 301 {
 302         struct stm32_port *stm32_port = to_stm32_port(port);
 303         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 304
 305         *sr = readl_relaxed(port->membase + ofs->isr);
 306         /* Get pending characters in RDR or FIFO */
 307         if (*sr & USART_SR_RXNE) {
 308                 /* Get all pending characters from the RDR or the FIFO when using interrupts */
 309                 if (!stm32_usart_rx_dma_enabled(port))
 310                         return true;
 311
 312                 /* Handle only RX data errors when using DMA */
 313                 if (*sr & USART_SR_ERR_MASK)
 314                         return true;
 315         }
 316
 317         return false;
 318 }
 319
 320 static unsigned long stm32_usart_get_char_pio(struct uart_port *port)
 321 {
 322         struct stm32_port *stm32_port = to_stm32_port(port);
 323         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 324         unsigned long c;
 325
 326         c = readl_relaxed(port->membase + ofs->rdr);
 327         /* Apply RDR data mask */
 328         c &= stm32_port->rdr_mask;
 329
 330         return c;
 331 }
 332
 333 static unsigned int stm32_usart_receive_chars_pio(struct uart_port *port)
 334 {
 335         struct stm32_port *stm32_port = to_stm32_port(port);
 336         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 337         unsigned long c;
 338         unsigned int size = 0;
 339         u32 sr;
 340         char flag;
 341
 342         while (stm32_usart_pending_rx_pio(port, &sr)) {
 343                 sr |= USART_SR_DUMMY_RX;
 344                 flag = TTY_NORMAL;
 345
 346                 /*
 347                  * Status bits has to be cleared before reading the RDR:
 348                  * In FIFO mode, reading the RDR will pop the next data
 349                  * (if any) along with its status bits into the SR.
 350                  * Not doing so leads to misalignement between RDR and SR,
 351                  * and clear status bits of the next rx data.
 352                  *
 353                  * Clear errors flags for stm32f7 and stm32h7 compatible
 354                  * devices. On stm32f4 compatible devices, the error bit is
 355                  * cleared by the sequence [read SR - read DR].
 356                  */
 357                 if ((sr & USART_SR_ERR_MASK) && ofs->icr != UNDEF_REG)
 358                         writel_relaxed(sr & USART_SR_ERR_MASK,
 359                                        port->membase + ofs->icr);
 360
 361                 c = stm32_usart_get_char_pio(port);
 362                 port->icount.rx++;
 363                 size++;
 364                 if (sr & USART_SR_ERR_MASK) {
 365                         if (sr & USART_SR_ORE) {
 366                                 port->icount.overrun++;
 367                         } else if (sr & USART_SR_PE) {
 368                                 port->icount.parity++;
 369                         } else if (sr & USART_SR_FE) {
 370                                 /* Break detection if character is null */
 371                                 if (!c) {
 372                                         port->icount.brk++;
 373                                         if (uart_handle_break(port))
 374                                                 continue;
 375                                 } else {
 376                                         port->icount.frame++;
 377                                 }
 378                         }
 379
 380                         sr &= port->read_status_mask;
 381
 382                         if (sr & USART_SR_PE) {
 383                                 flag = TTY_PARITY;
 384                         } else if (sr & USART_SR_FE) {
 385                                 if (!c)
 386                                         flag = TTY_BREAK;
 387                                 else
 388                                         flag = TTY_FRAME;
 389                         }
 390                 }
 391
 392                 if (uart_prepare_sysrq_char(port, c))
 393                         continue;
 394                 uart_insert_char(port, sr, USART_SR_ORE, c, flag);
 395         }
 396
 397         return size;
 398 }
 399
 400 static void stm32_usart_push_buffer_dma(struct uart_port *port, unsigned int dma_size)
 401 {
 402         struct stm32_port *stm32_port = to_stm32_port(port);
 403         struct tty_port *ttyport = &stm32_port->port.state->port;
 404         unsigned char *dma_start;
 405         int dma_count, i;
 406
 407         dma_start = stm32_port->rx_buf + (RX_BUF_L - stm32_port->last_res);
 408
 409         /*
 410          * Apply rdr_mask on buffer in order to mask parity bit.
 411          * This loop is useless in cs8 mode because DMA copies only
 412          * 8 bits and already ignores parity bit.
 413          */
 414         if (!(stm32_port->rdr_mask == (BIT(8) - 1)))
 415                 for (i = 0; i < dma_size; i++)
 416                         *(dma_start + i) &= stm32_port->rdr_mask;
 417
 418         dma_count = tty_insert_flip_string(ttyport, dma_start, dma_size);
 419         port->icount.rx += dma_count;
 420         if (dma_count != dma_size)
 421                 port->icount.buf_overrun++;
 422         stm32_port->last_res -= dma_count;
 423         if (stm32_port->last_res == 0)
 424                 stm32_port->last_res = RX_BUF_L;
 425 }
 426
 427 static unsigned int stm32_usart_receive_chars_dma(struct uart_port *port)
 428 {
 429         struct stm32_port *stm32_port = to_stm32_port(port);
 430         unsigned int dma_size, size = 0;
 431
 432         /* DMA buffer is configured in cyclic mode and handles the rollback of the buffer. */
 433         if (stm32_port->rx_dma_state.residue > stm32_port->last_res) {
 434                 /* Conditional first part: from last_res to end of DMA buffer */
 435                 dma_size = stm32_port->last_res;
 436                 stm32_usart_push_buffer_dma(port, dma_size);
 437                 size = dma_size;
 438         }
 439
 440         dma_size = stm32_port->last_res - stm32_port->rx_dma_state.residue;
 441         stm32_usart_push_buffer_dma(port, dma_size);
 442         size += dma_size;
 443
 444         return size;
 445 }
 446
 447 static unsigned int stm32_usart_receive_chars(struct uart_port *port, bool force_dma_flush)
 448 {
 449         struct stm32_port *stm32_port = to_stm32_port(port);
 450         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 451         enum dma_status rx_dma_status;
 452         u32 sr;
 453         unsigned int size = 0;
 454
 455         if (stm32_usart_rx_dma_enabled(port) || force_dma_flush) {
 456                 rx_dma_status = dmaengine_tx_status(stm32_port->rx_ch,
 457                                                     stm32_port->rx_ch->cookie,
 458                                                     &stm32_port->rx_dma_state);
 459                 if (rx_dma_status == DMA_IN_PROGRESS) {
 460                         /* Empty DMA buffer */
 461                         size = stm32_usart_receive_chars_dma(port);
 462                         sr = readl_relaxed(port->membase + ofs->isr);
 463                         if (sr & USART_SR_ERR_MASK) {
 464                                 /* Disable DMA request line */
 465                                 stm32_usart_clr_bits(port, ofs->cr3, USART_CR3_DMAR);
 466
 467                                 /* Switch to PIO mode to handle the errors */
 468                                 size += stm32_usart_receive_chars_pio(port);
 469
 470                                 /* Switch back to DMA mode */
 471                                 stm32_usart_set_bits(port, ofs->cr3, USART_CR3_DMAR);
 472                         }
 473                 } else {
 474                         /* Disable RX DMA */
 475                         dmaengine_terminate_async(stm32_port->rx_ch);
 476                         stm32_usart_clr_bits(port, ofs->cr3, USART_CR3_DMAR);
 477                         /* Fall back to interrupt mode */
 478                         dev_dbg(port->dev, "DMA error, fallback to irq mode\n");
 479                         size = stm32_usart_receive_chars_pio(port);
 480                 }
 481         } else {
 482                 size = stm32_usart_receive_chars_pio(port);
 483         }
 484
 485         return size;
 486 }
 487
 488 static void stm32_usart_tx_dma_terminate(struct stm32_port *stm32_port)
 489 {
 490         dmaengine_terminate_async(stm32_port->tx_ch);
 491         stm32_port->tx_dma_busy = false;
 492 }
 493
 494 static bool stm32_usart_tx_dma_started(struct stm32_port *stm32_port)
 495 {
 496         /*
 497          * We cannot use the function "dmaengine_tx_status" to know the
 498          * status of DMA. This function does not show if the "dma complete"
 499          * callback of the DMA transaction has been called. So we prefer
 500          * to use "tx_dma_busy" flag to prevent dual DMA transaction at the
 501          * same time.
 502          */
 503         return stm32_port->tx_dma_busy;
 504 }
 505
 506 static bool stm32_usart_tx_dma_enabled(struct stm32_port *stm32_port)
 507 {
 508         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 509
 510         return !!(readl_relaxed(stm32_port->port.membase + ofs->cr3) & USART_CR3_DMAT);
 511 }
 512
 513 static void stm32_usart_tx_dma_complete(void *arg)
 514 {
 515         struct uart_port *port = arg;
 516         struct stm32_port *stm32port = to_stm32_port(port);
 517         const struct stm32_usart_offsets *ofs = &stm32port->info->ofs;
 518         unsigned long flags;
 519
 520         stm32_usart_clr_bits(port, ofs->cr3, USART_CR3_DMAT);
 521         stm32_usart_tx_dma_terminate(stm32port);
 522
 523         /* Let's see if we have pending data to send */
 524         spin_lock_irqsave(&port->lock, flags);
 525         stm32_usart_transmit_chars(port);
 526         spin_unlock_irqrestore(&port->lock, flags);
 527 }
 528
 529 static void stm32_usart_tx_interrupt_enable(struct uart_port *port)
 530 {
 531         struct stm32_port *stm32_port = to_stm32_port(port);
 532         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 533
 534         /*
 535          * Enables TX FIFO threashold irq when FIFO is enabled,
 536          * or TX empty irq when FIFO is disabled
 537          */
 538         if (stm32_port->fifoen && stm32_port->txftcfg >= 0)
 539                 stm32_usart_set_bits(port, ofs->cr3, USART_CR3_TXFTIE);
 540         else
 541                 stm32_usart_set_bits(port, ofs->cr1, USART_CR1_TXEIE);
 542 }
 543
 544 static void stm32_usart_tc_interrupt_enable(struct uart_port *port)
 545 {
 546         struct stm32_port *stm32_port = to_stm32_port(port);
 547         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 548
 549         stm32_usart_set_bits(port, ofs->cr1, USART_CR1_TCIE);
 550 }
 551
 552 static void stm32_usart_rx_dma_complete(void *arg)
 553 {
 554         struct uart_port *port = arg;
 555         struct tty_port *tport = &port->state->port;
 556         unsigned int size;
 557         unsigned long flags;
 558
 559         spin_lock_irqsave(&port->lock, flags);
 560         size = stm32_usart_receive_chars(port, false);
 561         uart_unlock_and_check_sysrq_irqrestore(port, flags);
 562         if (size)
 563                 tty_flip_buffer_push(tport);
 564 }
 565
 566 static void stm32_usart_tx_interrupt_disable(struct uart_port *port)
 567 {
 568         struct stm32_port *stm32_port = to_stm32_port(port);
 569         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 570
 571         if (stm32_port->fifoen && stm32_port->txftcfg >= 0)
 572                 stm32_usart_clr_bits(port, ofs->cr3, USART_CR3_TXFTIE);
 573         else
 574                 stm32_usart_clr_bits(port, ofs->cr1, USART_CR1_TXEIE);
 575 }
 576
 577 static void stm32_usart_tc_interrupt_disable(struct uart_port *port)
 578 {
 579         struct stm32_port *stm32_port = to_stm32_port(port);
 580         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 581
 582         stm32_usart_clr_bits(port, ofs->cr1, USART_CR1_TCIE);
 583 }
 584
 585 static void stm32_usart_transmit_chars_pio(struct uart_port *port)
 586 {
 587         struct stm32_port *stm32_port = to_stm32_port(port);
 588         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 589         struct circ_buf *xmit = &port->state->xmit;
 590
 591         if (stm32_usart_tx_dma_enabled(stm32_port))
 592                 stm32_usart_clr_bits(port, ofs->cr3, USART_CR3_DMAT);
 593
 594         while (!uart_circ_empty(xmit)) {
 595                 /* Check that TDR is empty before filling FIFO */
 596                 if (!(readl_relaxed(port->membase + ofs->isr) & USART_SR_TXE))
 597                         break;
 598                 writel_relaxed(xmit->buf[xmit->tail], port->membase + ofs->tdr);
 599                 uart_xmit_advance(port, 1);
 600         }
 601
 602         /* rely on TXE irq (mask or unmask) for sending remaining data */
 603         if (uart_circ_empty(xmit))
 604                 stm32_usart_tx_interrupt_disable(port);
 605         else
 606                 stm32_usart_tx_interrupt_enable(port);
 607 }
 608
 609 static void stm32_usart_transmit_chars_dma(struct uart_port *port)
 610 {
 611         struct stm32_port *stm32port = to_stm32_port(port);
 612         const struct stm32_usart_offsets *ofs = &stm32port->info->ofs;
 613         struct circ_buf *xmit = &port->state->xmit;
 614         struct dma_async_tx_descriptor *desc = NULL;
 615         unsigned int count;
 616
 617         if (stm32_usart_tx_dma_started(stm32port)) {
 618                 if (!stm32_usart_tx_dma_enabled(stm32port))
 619                         stm32_usart_set_bits(port, ofs->cr3, USART_CR3_DMAT);
 620                 return;
 621         }
 622
 623         count = uart_circ_chars_pending(xmit);
 624
 625         if (count > TX_BUF_L)
 626                 count = TX_BUF_L;
 627
 628         if (xmit->tail < xmit->head) {
 629                 memcpy(&stm32port->tx_buf[0], &xmit->buf[xmit->tail], count);
 630         } else {
 631                 size_t one = UART_XMIT_SIZE - xmit->tail;
 632                 size_t two;
 633
 634                 if (one > count)
 635                         one = count;
 636                 two = count - one;
 637
 638                 memcpy(&stm32port->tx_buf[0], &xmit->buf[xmit->tail], one);
 639                 if (two)
 640                         memcpy(&stm32port->tx_buf[one], &xmit->buf[0], two);
 641         }
 642
 643         desc = dmaengine_prep_slave_single(stm32port->tx_ch,
 644                                            stm32port->tx_dma_buf,
 645                                            count,
 646                                            DMA_MEM_TO_DEV,
 647                                            DMA_PREP_INTERRUPT);
 648
 649         if (!desc)
 650                 goto fallback_err;
 651
 652         /*
 653          * Set "tx_dma_busy" flag. This flag will be released when
 654          * dmaengine_terminate_async will be called. This flag helps
 655          * transmit_chars_dma not to start another DMA transaction
 656          * if the callback of the previous is not yet called.
 657          */
 658         stm32port->tx_dma_busy = true;
 659
 660         desc->callback = stm32_usart_tx_dma_complete;
 661         desc->callback_param = port;
 662
 663         /* Push current DMA TX transaction in the pending queue */
 664         if (dma_submit_error(dmaengine_submit(desc))) {
 665                 /* dma no yet started, safe to free resources */
 666                 stm32_usart_tx_dma_terminate(stm32port);
 667                 goto fallback_err;
 668         }
 669
 670         /* Issue pending DMA TX requests */
 671         dma_async_issue_pending(stm32port->tx_ch);
 672
 673         stm32_usart_set_bits(port, ofs->cr3, USART_CR3_DMAT);
 674
 675         uart_xmit_advance(port, count);
 676
 677         return;
 678
 679 fallback_err:
 680         stm32_usart_transmit_chars_pio(port);
 681 }
 682
 683 static void stm32_usart_transmit_chars(struct uart_port *port)
 684 {
 685         struct stm32_port *stm32_port = to_stm32_port(port);
 686         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 687         struct circ_buf *xmit = &port->state->xmit;
 688         u32 isr;
 689         int ret;
 690
 691         if (!stm32_port->hw_flow_control &&
 692             port->rs485.flags & SER_RS485_ENABLED) {
 693                 stm32_port->txdone = false;
 694                 stm32_usart_tc_interrupt_disable(port);
 695                 stm32_usart_rs485_rts_enable(port);
 696         }
 697
 698         if (port->x_char) {
 699                 if (stm32_usart_tx_dma_started(stm32_port) &&
 700                     stm32_usart_tx_dma_enabled(stm32_port))
 701                         stm32_usart_clr_bits(port, ofs->cr3, USART_CR3_DMAT);
 702
 703                 /* Check that TDR is empty before filling FIFO */
 704                 ret =
 705                 readl_relaxed_poll_timeout_atomic(port->membase + ofs->isr,
 706                                                   isr,
 707                                                   (isr & USART_SR_TXE),
 708                                                   10, 1000);
 709                 if (ret)
 710                         dev_warn(port->dev, "1 character may be erased\n");
 711
 712                 writel_relaxed(port->x_char, port->membase + ofs->tdr);
 713                 port->x_char = 0;
 714                 port->icount.tx++;
 715                 if (stm32_usart_tx_dma_started(stm32_port))
 716                         stm32_usart_set_bits(port, ofs->cr3, USART_CR3_DMAT);
 717                 return;
 718         }
 719
 720         if (uart_circ_empty(xmit) || uart_tx_stopped(port)) {
 721                 stm32_usart_tx_interrupt_disable(port);
 722                 return;
 723         }
 724
 725         if (ofs->icr == UNDEF_REG)
 726                 stm32_usart_clr_bits(port, ofs->isr, USART_SR_TC);
 727         else
 728                 writel_relaxed(USART_ICR_TCCF, port->membase + ofs->icr);
 729
 730         if (stm32_port->tx_ch)
 731                 stm32_usart_transmit_chars_dma(port);
 732         else
 733                 stm32_usart_transmit_chars_pio(port);
 734
 735         if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
 736                 uart_write_wakeup(port);
 737
 738         if (uart_circ_empty(xmit)) {
 739                 stm32_usart_tx_interrupt_disable(port);
 740                 if (!stm32_port->hw_flow_control &&
 741                     port->rs485.flags & SER_RS485_ENABLED) {
 742                         stm32_port->txdone = true;
 743                         stm32_usart_tc_interrupt_enable(port);
 744                 }
 745         }
 746 }
 747
 748 static irqreturn_t stm32_usart_interrupt(int irq, void *ptr)
 749 {
 750         struct uart_port *port = ptr;
 751         struct tty_port *tport = &port->state->port;
 752         struct stm32_port *stm32_port = to_stm32_port(port);
 753         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 754         u32 sr;
 755         unsigned int size;
 756
 757         sr = readl_relaxed(port->membase + ofs->isr);
 758
 759         if (!stm32_port->hw_flow_control &&
 760             port->rs485.flags & SER_RS485_ENABLED &&
 761             (sr & USART_SR_TC)) {
 762                 stm32_usart_tc_interrupt_disable(port);
 763                 stm32_usart_rs485_rts_disable(port);
 764         }
 765
 766         if ((sr & USART_SR_RTOF) && ofs->icr != UNDEF_REG)
 767                 writel_relaxed(USART_ICR_RTOCF,
 768                                port->membase + ofs->icr);
 769
 770         if ((sr & USART_SR_WUF) && ofs->icr != UNDEF_REG) {
 771                 /* Clear wake up flag and disable wake up interrupt */
 772                 writel_relaxed(USART_ICR_WUCF,
 773                                port->membase + ofs->icr);
 774                 stm32_usart_clr_bits(port, ofs->cr3, USART_CR3_WUFIE);
 775                 if (irqd_is_wakeup_set(irq_get_irq_data(port->irq)))
 776                         pm_wakeup_event(tport->tty->dev, 0);
 777         }
 778
 779         /*
 780          * rx errors in dma mode has to be handled ASAP to avoid overrun as the DMA request
 781          * line has been masked by HW and rx data are stacking in FIFO.
 782          */
 783         if (!stm32_port->throttled) {
 784                 if (((sr & USART_SR_RXNE) && !stm32_usart_rx_dma_enabled(port)) ||
 785                     ((sr & USART_SR_ERR_MASK) && stm32_usart_rx_dma_enabled(port))) {
 786                         spin_lock(&port->lock);
 787                         size = stm32_usart_receive_chars(port, false);
 788                         uart_unlock_and_check_sysrq(port);
 789                         if (size)
 790                                 tty_flip_buffer_push(tport);
 791                 }
 792         }
 793
 794         if ((sr & USART_SR_TXE) && !(stm32_port->tx_ch)) {
 795                 spin_lock(&port->lock);
 796                 stm32_usart_transmit_chars(port);
 797                 spin_unlock(&port->lock);
 798         }
 799
 800         if (stm32_usart_rx_dma_enabled(port))
 801                 return IRQ_WAKE_THREAD;
 802         else
 803                 return IRQ_HANDLED;
 804 }
 805
 806 static irqreturn_t stm32_usart_threaded_interrupt(int irq, void *ptr)
 807 {
 808         struct uart_port *port = ptr;
 809         struct tty_port *tport = &port->state->port;
 810         struct stm32_port *stm32_port = to_stm32_port(port);
 811         unsigned int size;
 812         unsigned long flags;
 813
 814         /* Receiver timeout irq for DMA RX */
 815         if (!stm32_port->throttled) {
 816                 spin_lock_irqsave(&port->lock, flags);
 817                 size = stm32_usart_receive_chars(port, false);
 818                 uart_unlock_and_check_sysrq_irqrestore(port, flags);
 819                 if (size)
 820                         tty_flip_buffer_push(tport);
 821         }
 822
 823         return IRQ_HANDLED;
 824 }
 825
 826 static void stm32_usart_set_mctrl(struct uart_port *port, unsigned int mctrl)
 827 {
 828         struct stm32_port *stm32_port = to_stm32_port(port);
 829         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 830
 831         if ((mctrl & TIOCM_RTS) && (port->status & UPSTAT_AUTORTS))
 832                 stm32_usart_set_bits(port, ofs->cr3, USART_CR3_RTSE);
 833         else
 834                 stm32_usart_clr_bits(port, ofs->cr3, USART_CR3_RTSE);
 835
 836         mctrl_gpio_set(stm32_port->gpios, mctrl);
 837 }
 838
 839 static unsigned int stm32_usart_get_mctrl(struct uart_port *port)
 840 {
 841         struct stm32_port *stm32_port = to_stm32_port(port);
 842         unsigned int ret;
 843
 844         /* This routine is used to get signals of: DCD, DSR, RI, and CTS */
 845         ret = TIOCM_CAR | TIOCM_DSR | TIOCM_CTS;
 846
 847         return mctrl_gpio_get(stm32_port->gpios, &ret);
 848 }
 849
 850 static void stm32_usart_enable_ms(struct uart_port *port)
 851 {
 852         mctrl_gpio_enable_ms(to_stm32_port(port)->gpios);
 853 }
 854
 855 static void stm32_usart_disable_ms(struct uart_port *port)
 856 {
 857         mctrl_gpio_disable_ms(to_stm32_port(port)->gpios);
 858 }
 859
 860 /* Transmit stop */
 861 static void stm32_usart_stop_tx(struct uart_port *port)
 862 {
 863         struct stm32_port *stm32_port = to_stm32_port(port);
 864         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 865
 866         stm32_usart_tx_interrupt_disable(port);
 867         if (stm32_usart_tx_dma_started(stm32_port) && stm32_usart_tx_dma_enabled(stm32_port))
 868                 stm32_usart_clr_bits(port, ofs->cr3, USART_CR3_DMAT);
 869
 870         stm32_usart_rs485_rts_disable(port);
 871 }
 872
 873 /* There are probably characters waiting to be transmitted. */
 874 static void stm32_usart_start_tx(struct uart_port *port)
 875 {
 876         struct circ_buf *xmit = &port->state->xmit;
 877
 878         if (uart_circ_empty(xmit) && !port->x_char) {
 879                 stm32_usart_rs485_rts_disable(port);
 880                 return;
 881         }
 882
 883         stm32_usart_rs485_rts_enable(port);
 884
 885         stm32_usart_transmit_chars(port);
 886 }
 887
 888 /* Flush the transmit buffer. */
 889 static void stm32_usart_flush_buffer(struct uart_port *port)
 890 {
 891         struct stm32_port *stm32_port = to_stm32_port(port);
 892         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 893
 894         if (stm32_port->tx_ch) {
 895                 stm32_usart_tx_dma_terminate(stm32_port);
 896                 stm32_usart_clr_bits(port, ofs->cr3, USART_CR3_DMAT);
 897         }
 898 }
 899
 900 /* Throttle the remote when input buffer is about to overflow. */
 901 static void stm32_usart_throttle(struct uart_port *port)
 902 {
 903         struct stm32_port *stm32_port = to_stm32_port(port);
 904         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 905         unsigned long flags;
 906
 907         spin_lock_irqsave(&port->lock, flags);
 908
 909         /*
 910          * Disable DMA request line if enabled, so the RX data gets queued into the FIFO.
 911          * Hardware flow control is triggered when RX FIFO is full.
 912          */
 913         if (stm32_usart_rx_dma_enabled(port))
 914                 stm32_usart_clr_bits(port, ofs->cr3, USART_CR3_DMAR);
 915
 916         stm32_usart_clr_bits(port, ofs->cr1, stm32_port->cr1_irq);
 917         if (stm32_port->cr3_irq)
 918                 stm32_usart_clr_bits(port, ofs->cr3, stm32_port->cr3_irq);
 919
 920         stm32_port->throttled = true;
 921         spin_unlock_irqrestore(&port->lock, flags);
 922 }
 923
 924 /* Unthrottle the remote, the input buffer can now accept data. */
 925 static void stm32_usart_unthrottle(struct uart_port *port)
 926 {
 927         struct stm32_port *stm32_port = to_stm32_port(port);
 928         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 929         unsigned long flags;
 930
 931         spin_lock_irqsave(&port->lock, flags);
 932         stm32_usart_set_bits(port, ofs->cr1, stm32_port->cr1_irq);
 933         if (stm32_port->cr3_irq)
 934                 stm32_usart_set_bits(port, ofs->cr3, stm32_port->cr3_irq);
 935
 936         /*
 937          * Switch back to DMA mode (re-enable DMA request line).
 938          * Hardware flow control is stopped when FIFO is not full any more.
 939          */
 940         if (stm32_port->rx_ch)
 941                 stm32_usart_set_bits(port, ofs->cr3, USART_CR3_DMAR);
 942
 943         stm32_port->throttled = false;
 944         spin_unlock_irqrestore(&port->lock, flags);
 945 }
 946
 947 /* Receive stop */
 948 static void stm32_usart_stop_rx(struct uart_port *port)
 949 {
 950         struct stm32_port *stm32_port = to_stm32_port(port);
 951         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 952
 953         /* Disable DMA request line. */
 954         if (stm32_port->rx_ch)
 955                 stm32_usart_clr_bits(port, ofs->cr3, USART_CR3_DMAR);
 956
 957         stm32_usart_clr_bits(port, ofs->cr1, stm32_port->cr1_irq);
 958         if (stm32_port->cr3_irq)
 959                 stm32_usart_clr_bits(port, ofs->cr3, stm32_port->cr3_irq);
 960 }
 961
 962 /* Handle breaks - ignored by us */
 963 static void stm32_usart_break_ctl(struct uart_port *port, int break_state)
 964 {
 965 }
 966
 967 static int stm32_usart_start_rx_dma_cyclic(struct uart_port *port)
 968 {
 969         struct stm32_port *stm32_port = to_stm32_port(port);
 970         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 971         struct dma_async_tx_descriptor *desc;
 972         int ret;
 973
 974         stm32_port->last_res = RX_BUF_L;
 975         /* Prepare a DMA cyclic transaction */
 976         desc = dmaengine_prep_dma_cyclic(stm32_port->rx_ch,
 977                                          stm32_port->rx_dma_buf,
 978                                          RX_BUF_L, RX_BUF_P,
 979                                          DMA_DEV_TO_MEM,
 980                                          DMA_PREP_INTERRUPT);
 981         if (!desc) {
 982                 dev_err(port->dev, "rx dma prep cyclic failed\n");
 983                 return -ENODEV;
 984         }
 985
 986         desc->callback = stm32_usart_rx_dma_complete;
 987         desc->callback_param = port;
 988
 989         /* Push current DMA transaction in the pending queue */
 990         ret = dma_submit_error(dmaengine_submit(desc));
 991         if (ret) {
 992                 dmaengine_terminate_sync(stm32_port->rx_ch);
 993                 return ret;
 994         }
 995
 996         /* Issue pending DMA requests */
 997         dma_async_issue_pending(stm32_port->rx_ch);
 998
 999         /*
1000          * DMA request line not re-enabled at resume when port is throttled.
1001          * It will be re-enabled by unthrottle ops.
1002          */
1003         if (!stm32_port->throttled)
1004                 stm32_usart_set_bits(port, ofs->cr3, USART_CR3_DMAR);
1005
1006         return 0;
1007 }
1008
1009 static int stm32_usart_startup(struct uart_port *port)
1010 {
1011         struct stm32_port *stm32_port = to_stm32_port(port);
1012         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
1013         const struct stm32_usart_config *cfg = &stm32_port->info->cfg;
1014         const char *name = to_platform_device(port->dev)->name;
1015         u32 val;
1016         int ret;
1017
1018         ret = request_threaded_irq(port->irq, stm32_usart_interrupt,
1019                                    stm32_usart_threaded_interrupt,
1020                                    IRQF_ONESHOT | IRQF_NO_SUSPEND,
1021                                    name, port);
1022         if (ret)
1023                 return ret;
1024
1025         if (stm32_port->swap) {
1026                 val = readl_relaxed(port->membase + ofs->cr2);
1027                 val |= USART_CR2_SWAP;
1028                 writel_relaxed(val, port->membase + ofs->cr2);
1029         }
1030
1031         /* RX FIFO Flush */
1032         if (ofs->rqr != UNDEF_REG)
1033                 writel_relaxed(USART_RQR_RXFRQ, port->membase + ofs->rqr);
1034
1035         if (stm32_port->rx_ch) {
1036                 ret = stm32_usart_start_rx_dma_cyclic(port);
1037                 if (ret) {
1038                         free_irq(port->irq, port);
1039                         return ret;
1040                 }
1041         }
1042
1043         /* RX enabling */
1044         val = stm32_port->cr1_irq | USART_CR1_RE | BIT(cfg->uart_enable_bit);
1045         stm32_usart_set_bits(port, ofs->cr1, val);
1046
1047         return 0;
1048 }
1049
1050 static void stm32_usart_shutdown(struct uart_port *port)
1051 {
1052         struct stm32_port *stm32_port = to_stm32_port(port);
1053         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
1054         const struct stm32_usart_config *cfg = &stm32_port->info->cfg;
1055         u32 val, isr;
1056         int ret;
1057
1058         if (stm32_usart_tx_dma_enabled(stm32_port))
1059                 stm32_usart_clr_bits(port, ofs->cr3, USART_CR3_DMAT);
1060
1061         if (stm32_usart_tx_dma_started(stm32_port))
1062                 stm32_usart_tx_dma_terminate(stm32_port);
1063
1064         /* Disable modem control interrupts */
1065         stm32_usart_disable_ms(port);
1066
1067         val = USART_CR1_TXEIE | USART_CR1_TE;
1068         val |= stm32_port->cr1_irq | USART_CR1_RE;
1069         val |= BIT(cfg->uart_enable_bit);
1070         if (stm32_port->fifoen)
1071                 val |= USART_CR1_FIFOEN;
1072
1073         ret = readl_relaxed_poll_timeout(port->membase + ofs->isr,
1074                                          isr, (isr & USART_SR_TC),
1075                                          10, 100000);
1076
1077         /* Send the TC error message only when ISR_TC is not set */
1078         if (ret)
1079                 dev_err(port->dev, "Transmission is not complete\n");
1080
1081         /* Disable RX DMA. */
1082         if (stm32_port->rx_ch)
1083                 dmaengine_terminate_async(stm32_port->rx_ch);
1084
1085         /* flush RX & TX FIFO */
1086         if (ofs->rqr != UNDEF_REG)
1087                 writel_relaxed(USART_RQR_TXFRQ | USART_RQR_RXFRQ,
1088                                port->membase + ofs->rqr);
1089
1090         stm32_usart_clr_bits(port, ofs->cr1, val);
1091
1092         free_irq(port->irq, port);
1093 }
1094
1095 static void stm32_usart_set_termios(struct uart_port *port,
1096                                     struct ktermios *termios,
1097                                     const struct ktermios *old)
1098 {
1099         struct stm32_port *stm32_port = to_stm32_port(port);
1100         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
1101         const struct stm32_usart_config *cfg = &stm32_port->info->cfg;
1102         struct serial_rs485 *rs485conf = &port->rs485;
1103         unsigned int baud, bits;
1104         u32 usartdiv, mantissa, fraction, oversampling;
1105         tcflag_t cflag = termios->c_cflag;
1106         u32 cr1, cr2, cr3, isr;
1107         unsigned long flags;
1108         int ret;
1109
1110         if (!stm32_port->hw_flow_control)
1111                 cflag &= ~CRTSCTS;
1112
1113         baud = uart_get_baud_rate(port, termios, old, 0, port->uartclk / 8);
1114
1115         spin_lock_irqsave(&port->lock, flags);
1116
1117         ret = readl_relaxed_poll_timeout_atomic(port->membase + ofs->isr,
1118                                                 isr,
1119                                                 (isr & USART_SR_TC),
1120                                                 10, 100000);
1121
1122         /* Send the TC error message only when ISR_TC is not set. */
1123         if (ret)
1124                 dev_err(port->dev, "Transmission is not complete\n");
1125
1126         /* Stop serial port and reset value */
1127         writel_relaxed(0, port->membase + ofs->cr1);
1128
1129         /* flush RX & TX FIFO */
1130         if (ofs->rqr != UNDEF_REG)
1131                 writel_relaxed(USART_RQR_TXFRQ | USART_RQR_RXFRQ,
1132                                port->membase + ofs->rqr);
1133
1134         cr1 = USART_CR1_TE | USART_CR1_RE;
1135         if (stm32_port->fifoen)
1136                 cr1 |= USART_CR1_FIFOEN;
1137         cr2 = stm32_port->swap ? USART_CR2_SWAP : 0;
1138
1139         /* Tx and RX FIFO configuration */
1140         cr3 = readl_relaxed(port->membase + ofs->cr3);
1141         cr3 &= USART_CR3_TXFTIE | USART_CR3_RXFTIE;
1142         if (stm32_port->fifoen) {
1143                 if (stm32_port->txftcfg >= 0)
1144                         cr3 |= stm32_port->txftcfg << USART_CR3_TXFTCFG_SHIFT;
1145                 if (stm32_port->rxftcfg >= 0)
1146                         cr3 |= stm32_port->rxftcfg << USART_CR3_RXFTCFG_SHIFT;
1147         }
1148
1149         if (cflag & CSTOPB)
1150                 cr2 |= USART_CR2_STOP_2B;
1151
1152         bits = tty_get_char_size(cflag);
1153         stm32_port->rdr_mask = (BIT(bits) - 1);
1154
1155         if (cflag & PARENB) {
1156                 bits++;
1157                 cr1 |= USART_CR1_PCE;
1158         }
1159
1160         /*
1161          * Word length configuration:
1162          * CS8 + parity, 9 bits word aka [M1:M0] = 0b01
1163          * CS7 or (CS6 + parity), 7 bits word aka [M1:M0] = 0b10
1164          * CS8 or (CS7 + parity), 8 bits word aka [M1:M0] = 0b00
1165          * M0 and M1 already cleared by cr1 initialization.
1166          */
1167         if (bits == 9) {
1168                 cr1 |= USART_CR1_M0;
1169         } else if ((bits == 7) && cfg->has_7bits_data) {
1170                 cr1 |= USART_CR1_M1;
1171         } else if (bits != 8) {
1172                 dev_dbg(port->dev, "Unsupported data bits config: %u bits\n"
1173                         , bits);
1174                 cflag &= ~CSIZE;
1175                 cflag |= CS8;
1176                 termios->c_cflag = cflag;
1177                 bits = 8;
1178                 if (cflag & PARENB) {
1179                         bits++;
1180                         cr1 |= USART_CR1_M0;
1181                 }
1182         }
1183
1184         if (ofs->rtor != UNDEF_REG && (stm32_port->rx_ch ||
1185                                        (stm32_port->fifoen &&
1186                                         stm32_port->rxftcfg >= 0))) {
1187                 if (cflag & CSTOPB)
1188                         bits = bits + 3; /* 1 start bit + 2 stop bits */
1189                 else
1190                         bits = bits + 2; /* 1 start bit + 1 stop bit */
1191
1192                 /* RX timeout irq to occur after last stop bit + bits */
1193                 stm32_port->cr1_irq = USART_CR1_RTOIE;
1194                 writel_relaxed(bits, port->membase + ofs->rtor);
1195                 cr2 |= USART_CR2_RTOEN;
1196                 /*
1197                  * Enable fifo threshold irq in two cases, either when there is no DMA, or when
1198                  * wake up over usart, from low power until the DMA gets re-enabled by resume.
1199                  */
1200                 stm32_port->cr3_irq =  USART_CR3_RXFTIE;
1201         }
1202
1203         cr1 |= stm32_port->cr1_irq;
1204         cr3 |= stm32_port->cr3_irq;
1205
1206         if (cflag & PARODD)
1207                 cr1 |= USART_CR1_PS;
1208
1209         port->status &= ~(UPSTAT_AUTOCTS | UPSTAT_AUTORTS);
1210         if (cflag & CRTSCTS) {
1211                 port->status |= UPSTAT_AUTOCTS | UPSTAT_AUTORTS;
1212                 cr3 |= USART_CR3_CTSE | USART_CR3_RTSE;
1213         }
1214
1215         usartdiv = DIV_ROUND_CLOSEST(port->uartclk, baud);
1216
1217         /*
1218          * The USART supports 16 or 8 times oversampling.
1219          * By default we prefer 16 times oversampling, so that the receiver
1220          * has a better tolerance to clock deviations.
1221          * 8 times oversampling is only used to achieve higher speeds.
1222          */
1223         if (usartdiv < 16) {
1224                 oversampling = 8;
1225                 cr1 |= USART_CR1_OVER8;
1226                 stm32_usart_set_bits(port, ofs->cr1, USART_CR1_OVER8);
1227         } else {
1228                 oversampling = 16;
1229                 cr1 &= ~USART_CR1_OVER8;
1230                 stm32_usart_clr_bits(port, ofs->cr1, USART_CR1_OVER8);
1231         }
1232
1233         mantissa = (usartdiv / oversampling) << USART_BRR_DIV_M_SHIFT;
1234         fraction = usartdiv % oversampling;
1235         writel_relaxed(mantissa | fraction, port->membase + ofs->brr);
1236
1237         uart_update_timeout(port, cflag, baud);
1238
1239         port->read_status_mask = USART_SR_ORE;
1240         if (termios->c_iflag & INPCK)
1241                 port->read_status_mask |= USART_SR_PE | USART_SR_FE;
1242         if (termios->c_iflag & (IGNBRK | BRKINT | PARMRK))
1243                 port->read_status_mask |= USART_SR_FE;
1244
1245         /* Characters to ignore */
1246         port->ignore_status_mask = 0;
1247         if (termios->c_iflag & IGNPAR)
1248                 port->ignore_status_mask = USART_SR_PE | USART_SR_FE;
1249         if (termios->c_iflag & IGNBRK) {
1250                 port->ignore_status_mask |= USART_SR_FE;
1251                 /*
1252                  * If we're ignoring parity and break indicators,
1253                  * ignore overruns too (for real raw support).
1254                  */
1255                 if (termios->c_iflag & IGNPAR)
1256                         port->ignore_status_mask |= USART_SR_ORE;
1257         }
1258
1259         /* Ignore all characters if CREAD is not set */
1260         if ((termios->c_cflag & CREAD) == 0)
1261                 port->ignore_status_mask |= USART_SR_DUMMY_RX;
1262
1263         if (stm32_port->rx_ch) {
1264                 /*
1265                  * Setup DMA to collect only valid data and enable error irqs.
1266                  * This also enables break reception when using DMA.
1267                  */
1268                 cr1 |= USART_CR1_PEIE;
1269                 cr3 |= USART_CR3_EIE;
1270                 cr3 |= USART_CR3_DMAR;
1271                 cr3 |= USART_CR3_DDRE;
1272         }
1273
1274         if (rs485conf->flags & SER_RS485_ENABLED) {
1275                 stm32_usart_config_reg_rs485(&cr1, &cr3,
1276                                              rs485conf->delay_rts_before_send,
1277                                              rs485conf->delay_rts_after_send,
1278                                              baud);
1279                 if (rs485conf->flags & SER_RS485_RTS_ON_SEND) {
1280                         cr3 &= ~USART_CR3_DEP;
1281                         rs485conf->flags &= ~SER_RS485_RTS_AFTER_SEND;
1282                 } else {
1283                         cr3 |= USART_CR3_DEP;
1284                         rs485conf->flags |= SER_RS485_RTS_AFTER_SEND;
1285                 }
1286
1287         } else {
1288                 cr3 &= ~(USART_CR3_DEM | USART_CR3_DEP);
1289                 cr1 &= ~(USART_CR1_DEDT_MASK | USART_CR1_DEAT_MASK);
1290         }
1291
1292         /* Configure wake up from low power on start bit detection */
1293         if (stm32_port->wakeup_src) {
1294                 cr3 &= ~USART_CR3_WUS_MASK;
1295                 cr3 |= USART_CR3_WUS_START_BIT;
1296         }
1297
1298         writel_relaxed(cr3, port->membase + ofs->cr3);
1299         writel_relaxed(cr2, port->membase + ofs->cr2);
1300         writel_relaxed(cr1, port->membase + ofs->cr1);
1301
1302         stm32_usart_set_bits(port, ofs->cr1, BIT(cfg->uart_enable_bit));
1303         spin_unlock_irqrestore(&port->lock, flags);
1304
1305         /* Handle modem control interrupts */
1306         if (UART_ENABLE_MS(port, termios->c_cflag))
1307                 stm32_usart_enable_ms(port);
1308         else
1309                 stm32_usart_disable_ms(port);
1310 }
1311
1312 static const char *stm32_usart_type(struct uart_port *port)
1313 {
1314         return (port->type == PORT_STM32) ? DRIVER_NAME : NULL;
1315 }
1316
1317 static void stm32_usart_release_port(struct uart_port *port)
1318 {
1319 }
1320
1321 static int stm32_usart_request_port(struct uart_port *port)
1322 {
1323         return 0;
1324 }
1325
1326 static void stm32_usart_config_port(struct uart_port *port, int flags)
1327 {
1328         if (flags & UART_CONFIG_TYPE)
1329                 port->type = PORT_STM32;
1330 }
1331
1332 static int
1333 stm32_usart_verify_port(struct uart_port *port, struct serial_struct *ser)
1334 {
1335         /* No user changeable parameters */
1336         return -EINVAL;
1337 }
1338
1339 static void stm32_usart_pm(struct uart_port *port, unsigned int state,
1340                            unsigned int oldstate)
1341 {
1342         struct stm32_port *stm32port = container_of(port,
1343                         struct stm32_port, port);
1344         const struct stm32_usart_offsets *ofs = &stm32port->info->ofs;
1345         const struct stm32_usart_config *cfg = &stm32port->info->cfg;
1346         unsigned long flags;
1347
1348         switch (state) {
1349         case UART_PM_STATE_ON:
1350                 pm_runtime_get_sync(port->dev);
1351                 break;
1352         case UART_PM_STATE_OFF:
1353                 spin_lock_irqsave(&port->lock, flags);
1354                 stm32_usart_clr_bits(port, ofs->cr1, BIT(cfg->uart_enable_bit));
1355                 spin_unlock_irqrestore(&port->lock, flags);
1356                 pm_runtime_put_sync(port->dev);
1357                 break;
1358         }
1359 }
1360
1361 #if defined(CONFIG_CONSOLE_POLL)
1362
1363  /* Callbacks for characters polling in debug context (i.e. KGDB). */
1364 static int stm32_usart_poll_init(struct uart_port *port)
1365 {
1366         struct stm32_port *stm32_port = to_stm32_port(port);
1367
1368         return clk_prepare_enable(stm32_port->clk);
1369 }
1370
1371 static int stm32_usart_poll_get_char(struct uart_port *port)
1372 {
1373         struct stm32_port *stm32_port = to_stm32_port(port);
1374         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
1375
1376         if (!(readl_relaxed(port->membase + ofs->isr) & USART_SR_RXNE))
1377                 return NO_POLL_CHAR;
1378
1379         return readl_relaxed(port->membase + ofs->rdr) & stm32_port->rdr_mask;
1380 }
1381
1382 static void stm32_usart_poll_put_char(struct uart_port *port, unsigned char ch)
1383 {
1384         stm32_usart_console_putchar(port, ch);
1385 }
1386 #endif /* CONFIG_CONSOLE_POLL */
1387
1388 static const struct uart_ops stm32_uart_ops = {
1389         .tx_empty       = stm32_usart_tx_empty,
1390         .set_mctrl      = stm32_usart_set_mctrl,
1391         .get_mctrl      = stm32_usart_get_mctrl,
1392         .stop_tx        = stm32_usart_stop_tx,
1393         .start_tx       = stm32_usart_start_tx,
1394         .throttle       = stm32_usart_throttle,
1395         .unthrottle     = stm32_usart_unthrottle,
1396         .stop_rx        = stm32_usart_stop_rx,
1397         .enable_ms      = stm32_usart_enable_ms,
1398         .break_ctl      = stm32_usart_break_ctl,
1399         .startup        = stm32_usart_startup,
1400         .shutdown       = stm32_usart_shutdown,
1401         .flush_buffer   = stm32_usart_flush_buffer,
1402         .set_termios    = stm32_usart_set_termios,
1403         .pm             = stm32_usart_pm,
1404         .type           = stm32_usart_type,
1405         .release_port   = stm32_usart_release_port,
1406         .request_port   = stm32_usart_request_port,
1407         .config_port    = stm32_usart_config_port,
1408         .verify_port    = stm32_usart_verify_port,
1409 #if defined(CONFIG_CONSOLE_POLL)
1410         .poll_init      = stm32_usart_poll_init,
1411         .poll_get_char  = stm32_usart_poll_get_char,
1412         .poll_put_char  = stm32_usart_poll_put_char,
1413 #endif /* CONFIG_CONSOLE_POLL */
1414 };
1415
1416 /*
1417  * STM32H7 RX & TX FIFO threshold configuration (CR3 RXFTCFG / TXFTCFG)
1418  * Note: 1 isn't a valid value in RXFTCFG / TXFTCFG. In this case,
1419  * RXNEIE / TXEIE can be used instead of threshold irqs: RXFTIE / TXFTIE.
1420  * So, RXFTCFG / TXFTCFG bitfields values are encoded as array index + 1.
1421  */
1422 static const u32 stm32h7_usart_fifo_thresh_cfg[] = { 1, 2, 4, 8, 12, 14, 16 };
1423
1424 static void stm32_usart_get_ftcfg(struct platform_device *pdev, const char *p,
1425                                   int *ftcfg)
1426 {
1427         u32 bytes, i;
1428
1429         /* DT option to get RX & TX FIFO threshold (default to 8 bytes) */
1430         if (of_property_read_u32(pdev->dev.of_node, p, &bytes))
1431                 bytes = 8;
1432
1433         for (i = 0; i < ARRAY_SIZE(stm32h7_usart_fifo_thresh_cfg); i++)
1434                 if (stm32h7_usart_fifo_thresh_cfg[i] >= bytes)
1435                         break;
1436         if (i >= ARRAY_SIZE(stm32h7_usart_fifo_thresh_cfg))
1437                 i = ARRAY_SIZE(stm32h7_usart_fifo_thresh_cfg) - 1;
1438
1439         dev_dbg(&pdev->dev, "%s set to %d bytes\n", p,
1440                 stm32h7_usart_fifo_thresh_cfg[i]);
1441
1442         /* Provide FIFO threshold ftcfg (1 is invalid: threshold irq unused) */
1443         if (i)
1444                 *ftcfg = i - 1;
1445         else
1446                 *ftcfg = -EINVAL;
1447 }
1448
1449 static void stm32_usart_deinit_port(struct stm32_port *stm32port)
1450 {
1451         clk_disable_unprepare(stm32port->clk);
1452 }
1453
1454 static const struct serial_rs485 stm32_rs485_supported = {
1455         .flags = SER_RS485_ENABLED | SER_RS485_RTS_ON_SEND | SER_RS485_RTS_AFTER_SEND |
1456                  SER_RS485_RX_DURING_TX,
1457         .delay_rts_before_send = 1,
1458         .delay_rts_after_send = 1,
1459 };
1460
1461 static int stm32_usart_init_port(struct stm32_port *stm32port,
1462                                  struct platform_device *pdev)
1463 {
1464         struct uart_port *port = &stm32port->port;
1465         struct resource *res;
1466         int ret, irq;
1467
1468         irq = platform_get_irq(pdev, 0);
1469         if (irq < 0)
1470                 return irq;
1471
1472         port->iotype    = UPIO_MEM;
1473         port->flags     = UPF_BOOT_AUTOCONF;
1474         port->ops       = &stm32_uart_ops;
1475         port->dev       = &pdev->dev;
1476         port->fifosize  = stm32port->info->cfg.fifosize;
1477         port->has_sysrq = IS_ENABLED(CONFIG_SERIAL_STM32_CONSOLE);
1478         port->irq = irq;
1479         port->rs485_config = stm32_usart_config_rs485;
1480         port->rs485_supported = stm32_rs485_supported;
1481
1482         ret = stm32_usart_init_rs485(port, pdev);
1483         if (ret)
1484                 return ret;
1485
1486         stm32port->wakeup_src = stm32port->info->cfg.has_wakeup &&
1487                 of_property_read_bool(pdev->dev.of_node, "wakeup-source");
1488
1489         stm32port->swap = stm32port->info->cfg.has_swap &&
1490                 of_property_read_bool(pdev->dev.of_node, "rx-tx-swap");
1491
1492         stm32port->fifoen = stm32port->info->cfg.has_fifo;
1493         if (stm32port->fifoen) {
1494                 stm32_usart_get_ftcfg(pdev, "rx-threshold",
1495                                       &stm32port->rxftcfg);
1496                 stm32_usart_get_ftcfg(pdev, "tx-threshold",
1497                                       &stm32port->txftcfg);
1498         }
1499
1500         port->membase = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
1501         if (IS_ERR(port->membase))
1502                 return PTR_ERR(port->membase);
1503         port->mapbase = res->start;
1504
1505         spin_lock_init(&port->lock);
1506
1507         stm32port->clk = devm_clk_get(&pdev->dev, NULL);
1508         if (IS_ERR(stm32port->clk))
1509                 return PTR_ERR(stm32port->clk);
1510
1511         /* Ensure that clk rate is correct by enabling the clk */
1512         ret = clk_prepare_enable(stm32port->clk);
1513         if (ret)
1514                 return ret;
1515
1516         stm32port->port.uartclk = clk_get_rate(stm32port->clk);
1517         if (!stm32port->port.uartclk) {
1518                 ret = -EINVAL;
1519                 goto err_clk;
1520         }
1521
1522         stm32port->gpios = mctrl_gpio_init(&stm32port->port, 0);
1523         if (IS_ERR(stm32port->gpios)) {
1524                 ret = PTR_ERR(stm32port->gpios);
1525                 goto err_clk;
1526         }
1527
1528         /*
1529          * Both CTS/RTS gpios and "st,hw-flow-ctrl" (deprecated) or "uart-has-rtscts"
1530          * properties should not be specified.
1531          */
1532         if (stm32port->hw_flow_control) {
1533                 if (mctrl_gpio_to_gpiod(stm32port->gpios, UART_GPIO_CTS) ||
1534                     mctrl_gpio_to_gpiod(stm32port->gpios, UART_GPIO_RTS)) {
1535                         dev_err(&pdev->dev, "Conflicting RTS/CTS config\n");
1536                         ret = -EINVAL;
1537                         goto err_clk;
1538                 }
1539         }
1540
1541         return ret;
1542
1543 err_clk:
1544         clk_disable_unprepare(stm32port->clk);
1545
1546         return ret;
1547 }
1548
1549 static struct stm32_port *stm32_usart_of_get_port(struct platform_device *pdev)
1550 {
1551         struct device_node *np = pdev->dev.of_node;
1552         int id;
1553
1554         if (!np)
1555                 return NULL;
1556
1557         id = of_alias_get_id(np, "serial");
1558         if (id < 0) {
1559                 dev_err(&pdev->dev, "failed to get alias id, errno %d\n", id);
1560                 return NULL;
1561         }
1562
1563         if (WARN_ON(id >= STM32_MAX_PORTS))
1564                 return NULL;
1565
1566         stm32_ports[id].hw_flow_control =
1567                 of_property_read_bool (np, "st,hw-flow-ctrl") /*deprecated*/ ||
1568                 of_property_read_bool (np, "uart-has-rtscts");
1569         stm32_ports[id].port.line = id;
1570         stm32_ports[id].cr1_irq = USART_CR1_RXNEIE;
1571         stm32_ports[id].cr3_irq = 0;
1572         stm32_ports[id].last_res = RX_BUF_L;
1573         return &stm32_ports[id];
1574 }
1575
1576 #ifdef CONFIG_OF
1577 static const struct of_device_id stm32_match[] = {
1578         { .compatible = "st,stm32-uart", .data = &stm32f4_info},
1579         { .compatible = "st,stm32f7-uart", .data = &stm32f7_info},
1580         { .compatible = "st,stm32h7-uart", .data = &stm32h7_info},
1581         {},
1582 };
1583
1584 MODULE_DEVICE_TABLE(of, stm32_match);
1585 #endif
1586
1587 static void stm32_usart_of_dma_rx_remove(struct stm32_port *stm32port,
1588                                          struct platform_device *pdev)
1589 {
1590         if (stm32port->rx_buf)
1591                 dma_free_coherent(&pdev->dev, RX_BUF_L, stm32port->rx_buf,
1592                                   stm32port->rx_dma_buf);
1593 }
1594
1595 static int stm32_usart_of_dma_rx_probe(struct stm32_port *stm32port,
1596                                        struct platform_device *pdev)
1597 {
1598         const struct stm32_usart_offsets *ofs = &stm32port->info->ofs;
1599         struct uart_port *port = &stm32port->port;
1600         struct device *dev = &pdev->dev;
1601         struct dma_slave_config config;
1602         int ret;
1603
1604         /*
1605          * Using DMA and threaded handler for the console could lead to
1606          * deadlocks.
1607          */
1608         if (uart_console(port))
1609                 return -ENODEV;
1610
1611         stm32port->rx_buf = dma_alloc_coherent(dev, RX_BUF_L,
1612                                                &stm32port->rx_dma_buf,
1613                                                GFP_KERNEL);
1614         if (!stm32port->rx_buf)
1615                 return -ENOMEM;
1616
1617         /* Configure DMA channel */
1618         memset(&config, 0, sizeof(config));
1619         config.src_addr = port->mapbase + ofs->rdr;
1620         config.src_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
1621
1622         ret = dmaengine_slave_config(stm32port->rx_ch, &config);
1623         if (ret < 0) {
1624                 dev_err(dev, "rx dma channel config failed\n");
1625                 stm32_usart_of_dma_rx_remove(stm32port, pdev);
1626                 return ret;
1627         }
1628
1629         return 0;
1630 }
1631
1632 static void stm32_usart_of_dma_tx_remove(struct stm32_port *stm32port,
1633                                          struct platform_device *pdev)
1634 {
1635         if (stm32port->tx_buf)
1636                 dma_free_coherent(&pdev->dev, TX_BUF_L, stm32port->tx_buf,
1637                                   stm32port->tx_dma_buf);
1638 }
1639
1640 static int stm32_usart_of_dma_tx_probe(struct stm32_port *stm32port,
1641                                        struct platform_device *pdev)
1642 {
1643         const struct stm32_usart_offsets *ofs = &stm32port->info->ofs;
1644         struct uart_port *port = &stm32port->port;
1645         struct device *dev = &pdev->dev;
1646         struct dma_slave_config config;
1647         int ret;
1648
1649         stm32port->tx_buf = dma_alloc_coherent(dev, TX_BUF_L,
1650                                                &stm32port->tx_dma_buf,
1651                                                GFP_KERNEL);
1652         if (!stm32port->tx_buf)
1653                 return -ENOMEM;
1654
1655         /* Configure DMA channel */
1656         memset(&config, 0, sizeof(config));
1657         config.dst_addr = port->mapbase + ofs->tdr;
1658         config.dst_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
1659
1660         ret = dmaengine_slave_config(stm32port->tx_ch, &config);
1661         if (ret < 0) {
1662                 dev_err(dev, "tx dma channel config failed\n");
1663                 stm32_usart_of_dma_tx_remove(stm32port, pdev);
1664                 return ret;
1665         }
1666
1667         return 0;
1668 }
1669
1670 static int stm32_usart_serial_probe(struct platform_device *pdev)
1671 {
1672         struct stm32_port *stm32port;
1673         int ret;
1674
1675         stm32port = stm32_usart_of_get_port(pdev);
1676         if (!stm32port)
1677                 return -ENODEV;
1678
1679         stm32port->info = of_device_get_match_data(&pdev->dev);
1680         if (!stm32port->info)
1681                 return -EINVAL;
1682
1683         stm32port->rx_ch = dma_request_chan(&pdev->dev, "rx");
1684         if (PTR_ERR(stm32port->rx_ch) == -EPROBE_DEFER)
1685                 return -EPROBE_DEFER;
1686
1687         /* Fall back in interrupt mode for any non-deferral error */
1688         if (IS_ERR(stm32port->rx_ch))
1689                 stm32port->rx_ch = NULL;
1690
1691         stm32port->tx_ch = dma_request_chan(&pdev->dev, "tx");
1692         if (PTR_ERR(stm32port->tx_ch) == -EPROBE_DEFER) {
1693                 ret = -EPROBE_DEFER;
1694                 goto err_dma_rx;
1695         }
1696         /* Fall back in interrupt mode for any non-deferral error */
1697         if (IS_ERR(stm32port->tx_ch))
1698                 stm32port->tx_ch = NULL;
1699
1700         ret = stm32_usart_init_port(stm32port, pdev);
1701         if (ret)
1702                 goto err_dma_tx;
1703
1704         if (stm32port->wakeup_src) {
1705                 device_set_wakeup_capable(&pdev->dev, true);
1706                 ret = dev_pm_set_wake_irq(&pdev->dev, stm32port->port.irq);
1707                 if (ret)
1708                         goto err_deinit_port;
1709         }
1710
1711         if (stm32port->rx_ch && stm32_usart_of_dma_rx_probe(stm32port, pdev)) {
1712                 /* Fall back in interrupt mode */
1713                 dma_release_channel(stm32port->rx_ch);
1714                 stm32port->rx_ch = NULL;
1715         }
1716
1717         if (stm32port->tx_ch && stm32_usart_of_dma_tx_probe(stm32port, pdev)) {
1718                 /* Fall back in interrupt mode */
1719                 dma_release_channel(stm32port->tx_ch);
1720                 stm32port->tx_ch = NULL;
1721         }
1722
1723         if (!stm32port->rx_ch)
1724                 dev_info(&pdev->dev, "interrupt mode for rx (no dma)\n");
1725         if (!stm32port->tx_ch)
1726                 dev_info(&pdev->dev, "interrupt mode for tx (no dma)\n");
1727
1728         platform_set_drvdata(pdev, &stm32port->port);
1729
1730         pm_runtime_get_noresume(&pdev->dev);
1731         pm_runtime_set_active(&pdev->dev);
1732         pm_runtime_enable(&pdev->dev);
1733
1734         ret = uart_add_one_port(&stm32_usart_driver, &stm32port->port);
1735         if (ret)
1736                 goto err_port;
1737
1738         pm_runtime_put_sync(&pdev->dev);
1739
1740         return 0;
1741
1742 err_port:
1743         pm_runtime_disable(&pdev->dev);
1744         pm_runtime_set_suspended(&pdev->dev);
1745         pm_runtime_put_noidle(&pdev->dev);
1746
1747         if (stm32port->tx_ch)
1748                 stm32_usart_of_dma_tx_remove(stm32port, pdev);
1749         if (stm32port->rx_ch)
1750                 stm32_usart_of_dma_rx_remove(stm32port, pdev);
1751
1752         if (stm32port->wakeup_src)
1753                 dev_pm_clear_wake_irq(&pdev->dev);
1754
1755 err_deinit_port:
1756         if (stm32port->wakeup_src)
1757                 device_set_wakeup_capable(&pdev->dev, false);
1758
1759         stm32_usart_deinit_port(stm32port);
1760
1761 err_dma_tx:
1762         if (stm32port->tx_ch)
1763                 dma_release_channel(stm32port->tx_ch);
1764
1765 err_dma_rx:
1766         if (stm32port->rx_ch)
1767                 dma_release_channel(stm32port->rx_ch);
1768
1769         return ret;
1770 }
1771
1772 static int stm32_usart_serial_remove(struct platform_device *pdev)
1773 {
1774         struct uart_port *port = platform_get_drvdata(pdev);
1775         struct stm32_port *stm32_port = to_stm32_port(port);
1776         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
1777         int err;
1778         u32 cr3;
1779
1780         pm_runtime_get_sync(&pdev->dev);
1781         err = uart_remove_one_port(&stm32_usart_driver, port);
1782         if (err)
1783                 return(err);
1784
1785         pm_runtime_disable(&pdev->dev);
1786         pm_runtime_set_suspended(&pdev->dev);
1787         pm_runtime_put_noidle(&pdev->dev);
1788
1789         stm32_usart_clr_bits(port, ofs->cr1, USART_CR1_PEIE);
1790         cr3 = readl_relaxed(port->membase + ofs->cr3);
1791         cr3 &= ~USART_CR3_EIE;
1792         cr3 &= ~USART_CR3_DMAR;
1793         cr3 &= ~USART_CR3_DDRE;
1794         writel_relaxed(cr3, port->membase + ofs->cr3);
1795
1796         if (stm32_port->tx_ch) {
1797                 stm32_usart_of_dma_tx_remove(stm32_port, pdev);
1798                 dma_release_channel(stm32_port->tx_ch);
1799         }
1800
1801         if (stm32_port->rx_ch) {
1802                 stm32_usart_of_dma_rx_remove(stm32_port, pdev);
1803                 dma_release_channel(stm32_port->rx_ch);
1804         }
1805
1806         stm32_usart_clr_bits(port, ofs->cr3, USART_CR3_DMAT);
1807
1808         if (stm32_port->wakeup_src) {
1809                 dev_pm_clear_wake_irq(&pdev->dev);
1810                 device_init_wakeup(&pdev->dev, false);
1811         }
1812
1813         stm32_usart_deinit_port(stm32_port);
1814
1815         return 0;
1816 }
1817
1818 static void __maybe_unused stm32_usart_console_putchar(struct uart_port *port, unsigned char ch)
1819 {
1820         struct stm32_port *stm32_port = to_stm32_port(port);
1821         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
1822         u32 isr;
1823         int ret;
1824
1825         ret = readl_relaxed_poll_timeout_atomic(port->membase + ofs->isr, isr,
1826                                                 (isr & USART_SR_TXE), 100,
1827                                                 STM32_USART_TIMEOUT_USEC);
1828         if (ret != 0) {
1829                 dev_err(port->dev, "Error while sending data in UART TX : %d\n", ret);
1830                 return;
1831         }
1832         writel_relaxed(ch, port->membase + ofs->tdr);
1833 }
1834
1835 #ifdef CONFIG_SERIAL_STM32_CONSOLE
1836 static void stm32_usart_console_write(struct console *co, const char *s,
1837                                       unsigned int cnt)
1838 {
1839         struct uart_port *port = &stm32_ports[co->index].port;
1840         struct stm32_port *stm32_port = to_stm32_port(port);
1841         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
1842         const struct stm32_usart_config *cfg = &stm32_port->info->cfg;
1843         unsigned long flags;
1844         u32 old_cr1, new_cr1;
1845         int locked = 1;
1846
1847         if (oops_in_progress)
1848                 locked = spin_trylock_irqsave(&port->lock, flags);
1849         else
1850                 spin_lock_irqsave(&port->lock, flags);
1851
1852         /* Save and disable interrupts, enable the transmitter */
1853         old_cr1 = readl_relaxed(port->membase + ofs->cr1);
1854         new_cr1 = old_cr1 & ~USART_CR1_IE_MASK;
1855         new_cr1 |=  USART_CR1_TE | BIT(cfg->uart_enable_bit);
1856         writel_relaxed(new_cr1, port->membase + ofs->cr1);
1857
1858         uart_console_write(port, s, cnt, stm32_usart_console_putchar);
1859
1860         /* Restore interrupt state */
1861         writel_relaxed(old_cr1, port->membase + ofs->cr1);
1862
1863         if (locked)
1864                 spin_unlock_irqrestore(&port->lock, flags);
1865 }
1866
1867 static int stm32_usart_console_setup(struct console *co, char *options)
1868 {
1869         struct stm32_port *stm32port;
1870         int baud = 9600;
1871         int bits = 8;
1872         int parity = 'n';
1873         int flow = 'n';
1874
1875         if (co->index >= STM32_MAX_PORTS)
1876                 return -ENODEV;
1877
1878         stm32port = &stm32_ports[co->index];
1879
1880         /*
1881          * This driver does not support early console initialization
1882          * (use ARM early printk support instead), so we only expect
1883          * this to be called during the uart port registration when the
1884          * driver gets probed and the port should be mapped at that point.
1885          */
1886         if (stm32port->port.mapbase == 0 || !stm32port->port.membase)
1887                 return -ENXIO;
1888
1889         if (options)
1890                 uart_parse_options(options, &baud, &parity, &bits, &flow);
1891
1892         return uart_set_options(&stm32port->port, co, baud, parity, bits, flow);
1893 }
1894
1895 static struct console stm32_console = {
1896         .name           = STM32_SERIAL_NAME,
1897         .device         = uart_console_device,
1898         .write          = stm32_usart_console_write,
1899         .setup          = stm32_usart_console_setup,
1900         .flags          = CON_PRINTBUFFER,
1901         .index          = -1,
1902         .data           = &stm32_usart_driver,
1903 };
1904
1905 #define STM32_SERIAL_CONSOLE (&stm32_console)
1906
1907 #else
1908 #define STM32_SERIAL_CONSOLE NULL
1909 #endif /* CONFIG_SERIAL_STM32_CONSOLE */
1910
1911 #ifdef CONFIG_SERIAL_EARLYCON
1912 static void early_stm32_usart_console_putchar(struct uart_port *port, unsigned char ch)
1913 {
1914         struct stm32_usart_info *info = port->private_data;
1915
1916         while (!(readl_relaxed(port->membase + info->ofs.isr) & USART_SR_TXE))
1917                 cpu_relax();
1918
1919         writel_relaxed(ch, port->membase + info->ofs.tdr);
1920 }
1921
1922 static void early_stm32_serial_write(struct console *console, const char *s, unsigned int count)
1923 {
1924         struct earlycon_device *device = console->data;
1925         struct uart_port *port = &device->port;
1926
1927         uart_console_write(port, s, count, early_stm32_usart_console_putchar);
1928 }
1929
1930 static int __init early_stm32_h7_serial_setup(struct earlycon_device *device, const char *options)
1931 {
1932         if (!(device->port.membase || device->port.iobase))
1933                 return -ENODEV;
1934         device->port.private_data = &stm32h7_info;
1935         device->con->write = early_stm32_serial_write;
1936         return 0;
1937 }
1938
1939 static int __init early_stm32_f7_serial_setup(struct earlycon_device *device, const char *options)
1940 {
1941         if (!(device->port.membase || device->port.iobase))
1942                 return -ENODEV;
1943         device->port.private_data = &stm32f7_info;
1944         device->con->write = early_stm32_serial_write;
1945         return 0;
1946 }
1947
1948 static int __init early_stm32_f4_serial_setup(struct earlycon_device *device, const char *options)
1949 {
1950         if (!(device->port.membase || device->port.iobase))
1951                 return -ENODEV;
1952         device->port.private_data = &stm32f4_info;
1953         device->con->write = early_stm32_serial_write;
1954         return 0;
1955 }
1956
1957 OF_EARLYCON_DECLARE(stm32, "st,stm32h7-uart", early_stm32_h7_serial_setup);
1958 OF_EARLYCON_DECLARE(stm32, "st,stm32f7-uart", early_stm32_f7_serial_setup);
1959 OF_EARLYCON_DECLARE(stm32, "st,stm32-uart", early_stm32_f4_serial_setup);
1960 #endif /* CONFIG_SERIAL_EARLYCON */
1961
1962 static struct uart_driver stm32_usart_driver = {
1963         .driver_name    = DRIVER_NAME,
1964         .dev_name       = STM32_SERIAL_NAME,
1965         .major          = 0,
1966         .minor          = 0,
1967         .nr             = STM32_MAX_PORTS,
1968         .cons           = STM32_SERIAL_CONSOLE,
1969 };
1970
1971 static int __maybe_unused stm32_usart_serial_en_wakeup(struct uart_port *port,
1972                                                        bool enable)
1973 {
1974         struct stm32_port *stm32_port = to_stm32_port(port);
1975         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
1976         struct tty_port *tport = &port->state->port;
1977         int ret;
1978         unsigned int size;
1979         unsigned long flags;
1980
1981         if (!stm32_port->wakeup_src || !tty_port_initialized(tport))
1982                 return 0;
1983
1984         /*
1985          * Enable low-power wake-up and wake-up irq if argument is set to
1986          * "enable", disable low-power wake-up and wake-up irq otherwise
1987          */
1988         if (enable) {
1989                 stm32_usart_set_bits(port, ofs->cr1, USART_CR1_UESM);
1990                 stm32_usart_set_bits(port, ofs->cr3, USART_CR3_WUFIE);
1991                 mctrl_gpio_enable_irq_wake(stm32_port->gpios);
1992
1993                 /*
1994                  * When DMA is used for reception, it must be disabled before
1995                  * entering low-power mode and re-enabled when exiting from
1996                  * low-power mode.
1997                  */
1998                 if (stm32_port->rx_ch) {
1999                         spin_lock_irqsave(&port->lock, flags);
2000                         /* Avoid race with RX IRQ when DMAR is cleared */
2001                         stm32_usart_clr_bits(port, ofs->cr3, USART_CR3_DMAR);
2002                         /* Poll data from DMA RX buffer if any */
2003                         size = stm32_usart_receive_chars(port, true);
2004                         dmaengine_terminate_async(stm32_port->rx_ch);
2005                         uart_unlock_and_check_sysrq_irqrestore(port, flags);
2006                         if (size)
2007                                 tty_flip_buffer_push(tport);
2008                 }
2009
2010                 /* Poll data from RX FIFO if any */
2011                 stm32_usart_receive_chars(port, false);
2012         } else {
2013                 if (stm32_port->rx_ch) {
2014                         ret = stm32_usart_start_rx_dma_cyclic(port);
2015                         if (ret)
2016                                 return ret;
2017                 }
2018                 mctrl_gpio_disable_irq_wake(stm32_port->gpios);
2019                 stm32_usart_clr_bits(port, ofs->cr1, USART_CR1_UESM);
2020                 stm32_usart_clr_bits(port, ofs->cr3, USART_CR3_WUFIE);
2021         }
2022
2023         return 0;
2024 }
2025
2026 static int __maybe_unused stm32_usart_serial_suspend(struct device *dev)
2027 {
2028         struct uart_port *port = dev_get_drvdata(dev);
2029         int ret;
2030
2031         uart_suspend_port(&stm32_usart_driver, port);
2032
2033         if (device_may_wakeup(dev) || device_wakeup_path(dev)) {
2034                 ret = stm32_usart_serial_en_wakeup(port, true);
2035                 if (ret)
2036                         return ret;
2037         }
2038
2039         /*
2040          * When "no_console_suspend" is enabled, keep the pinctrl default state
2041          * and rely on bootloader stage to restore this state upon resume.
2042          * Otherwise, apply the idle or sleep states depending on wakeup
2043          * capabilities.
2044          */
2045         if (console_suspend_enabled || !uart_console(port)) {
2046                 if (device_may_wakeup(dev) || device_wakeup_path(dev))
2047                         pinctrl_pm_select_idle_state(dev);
2048                 else
2049                         pinctrl_pm_select_sleep_state(dev);
2050         }
2051
2052         return 0;
2053 }
2054
2055 static int __maybe_unused stm32_usart_serial_resume(struct device *dev)
2056 {
2057         struct uart_port *port = dev_get_drvdata(dev);
2058         int ret;
2059
2060         pinctrl_pm_select_default_state(dev);
2061
2062         if (device_may_wakeup(dev) || device_wakeup_path(dev)) {
2063                 ret = stm32_usart_serial_en_wakeup(port, false);
2064                 if (ret)
2065                         return ret;
2066         }
2067
2068         return uart_resume_port(&stm32_usart_driver, port);
2069 }
2070
2071 static int __maybe_unused stm32_usart_runtime_suspend(struct device *dev)
2072 {
2073         struct uart_port *port = dev_get_drvdata(dev);
2074         struct stm32_port *stm32port = container_of(port,
2075                         struct stm32_port, port);
2076
2077         clk_disable_unprepare(stm32port->clk);
2078
2079         return 0;
2080 }
2081
2082 static int __maybe_unused stm32_usart_runtime_resume(struct device *dev)
2083 {
2084         struct uart_port *port = dev_get_drvdata(dev);
2085         struct stm32_port *stm32port = container_of(port,
2086                         struct stm32_port, port);
2087
2088         return clk_prepare_enable(stm32port->clk);
2089 }
2090
2091 static const struct dev_pm_ops stm32_serial_pm_ops = {
2092         SET_RUNTIME_PM_OPS(stm32_usart_runtime_suspend,
2093                            stm32_usart_runtime_resume, NULL)
2094         SET_SYSTEM_SLEEP_PM_OPS(stm32_usart_serial_suspend,
2095                                 stm32_usart_serial_resume)
2096 };
2097
2098 static struct platform_driver stm32_serial_driver = {
2099         .probe          = stm32_usart_serial_probe,
2100         .remove         = stm32_usart_serial_remove,
2101         .driver = {
2102                 .name   = DRIVER_NAME,
2103                 .pm     = &stm32_serial_pm_ops,
2104                 .of_match_table = of_match_ptr(stm32_match),
2105         },
2106 };
2107
2108 static int __init stm32_usart_init(void)
2109 {
2110         static char banner[] __initdata = "STM32 USART driver initialized";
2111         int ret;
2112
2113         pr_info("%s\n", banner);
2114
2115         ret = uart_register_driver(&stm32_usart_driver);
2116         if (ret)
2117                 return ret;
2118
2119         ret = platform_driver_register(&stm32_serial_driver);
2120         if (ret)
2121                 uart_unregister_driver(&stm32_usart_driver);
2122
2123         return ret;
2124 }
2125
2126 static void __exit stm32_usart_exit(void)
2127 {
2128         platform_driver_unregister(&stm32_serial_driver);
2129         uart_unregister_driver(&stm32_usart_driver);
2130 }
2131
2132 module_init(stm32_usart_init);
2133 module_exit(stm32_usart_exit);
2134
2135 MODULE_ALIAS("platform:" DRIVER_NAME);
2136 MODULE_DESCRIPTION("STMicroelectronics STM32 serial port driver");
2137 MODULE_LICENSE("GPL v2");