drivers/tty/serial/stm32-usart.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) Maxime Coquelin 2015
   4  * Copyright (C) STMicroelectronics SA 2017
   5  * Authors:  Maxime Coquelin <mcoquelin.stm32@gmail.com>
   6  *           Gerald Baeza <gerald.baeza@foss.st.com>
   7  *           Erwan Le Ray <erwan.leray@foss.st.com>
   8  *
   9  * Inspired by st-asc.c from STMicroelectronics (c)
  10  */
  11
  12 #include <linux/clk.h>
  13 #include <linux/console.h>
  14 #include <linux/delay.h>
  15 #include <linux/dma-direction.h>
  16 #include <linux/dmaengine.h>
  17 #include <linux/dma-mapping.h>
  18 #include <linux/io.h>
  19 #include <linux/iopoll.h>
  20 #include <linux/irq.h>
  21 #include <linux/module.h>
  22 #include <linux/of.h>
  23 #include <linux/of_platform.h>
  24 #include <linux/pinctrl/consumer.h>
  25 #include <linux/platform_device.h>
  26 #include <linux/pm_runtime.h>
  27 #include <linux/pm_wakeirq.h>
  28 #include <linux/serial_core.h>
  29 #include <linux/serial.h>
  30 #include <linux/spinlock.h>
  31 #include <linux/sysrq.h>
  32 #include <linux/tty_flip.h>
  33 #include <linux/tty.h>
  34
  35 #include "serial_mctrl_gpio.h"
  36 #include "stm32-usart.h"
  37
  38
  39 /* Register offsets */
  40 static struct stm32_usart_info __maybe_unused stm32f4_info = {
  41         .ofs = {
  42                 .isr    = 0x00,
  43                 .rdr    = 0x04,
  44                 .tdr    = 0x04,
  45                 .brr    = 0x08,
  46                 .cr1    = 0x0c,
  47                 .cr2    = 0x10,
  48                 .cr3    = 0x14,
  49                 .gtpr   = 0x18,
  50                 .rtor   = UNDEF_REG,
  51                 .rqr    = UNDEF_REG,
  52                 .icr    = UNDEF_REG,
  53         },
  54         .cfg = {
  55                 .uart_enable_bit = 13,
  56                 .has_7bits_data = false,
  57                 .fifosize = 1,
  58         }
  59 };
  60
  61 static struct stm32_usart_info __maybe_unused stm32f7_info = {
  62         .ofs = {
  63                 .cr1    = 0x00,
  64                 .cr2    = 0x04,
  65                 .cr3    = 0x08,
  66                 .brr    = 0x0c,
  67                 .gtpr   = 0x10,
  68                 .rtor   = 0x14,
  69                 .rqr    = 0x18,
  70                 .isr    = 0x1c,
  71                 .icr    = 0x20,
  72                 .rdr    = 0x24,
  73                 .tdr    = 0x28,
  74         },
  75         .cfg = {
  76                 .uart_enable_bit = 0,
  77                 .has_7bits_data = true,
  78                 .has_swap = true,
  79                 .fifosize = 1,
  80         }
  81 };
  82
  83 static struct stm32_usart_info __maybe_unused stm32h7_info = {
  84         .ofs = {
  85                 .cr1    = 0x00,
  86                 .cr2    = 0x04,
  87                 .cr3    = 0x08,
  88                 .brr    = 0x0c,
  89                 .gtpr   = 0x10,
  90                 .rtor   = 0x14,
  91                 .rqr    = 0x18,
  92                 .isr    = 0x1c,
  93                 .icr    = 0x20,
  94                 .rdr    = 0x24,
  95                 .tdr    = 0x28,
  96         },
  97         .cfg = {
  98                 .uart_enable_bit = 0,
  99                 .has_7bits_data = true,
 100                 .has_swap = true,
 101                 .has_wakeup = true,
 102                 .has_fifo = true,
 103                 .fifosize = 16,
 104         }
 105 };
 106
 107 static void stm32_usart_stop_tx(struct uart_port *port);
 108 static void stm32_usart_transmit_chars(struct uart_port *port);
 109 static void __maybe_unused stm32_usart_console_putchar(struct uart_port *port, unsigned char ch);
 110
 111 static inline struct stm32_port *to_stm32_port(struct uart_port *port)
 112 {
 113         return container_of(port, struct stm32_port, port);
 114 }
 115
 116 static void stm32_usart_set_bits(struct uart_port *port, u32 reg, u32 bits)
 117 {
 118         u32 val;
 119
 120         val = readl_relaxed(port->membase + reg);
 121         val |= bits;
 122         writel_relaxed(val, port->membase + reg);
 123 }
 124
 125 static void stm32_usart_clr_bits(struct uart_port *port, u32 reg, u32 bits)
 126 {
 127         u32 val;
 128
 129         val = readl_relaxed(port->membase + reg);
 130         val &= ~bits;
 131         writel_relaxed(val, port->membase + reg);
 132 }
 133
 134 static unsigned int stm32_usart_tx_empty(struct uart_port *port)
 135 {
 136         struct stm32_port *stm32_port = to_stm32_port(port);
 137         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 138
 139         if (readl_relaxed(port->membase + ofs->isr) & USART_SR_TC)
 140                 return TIOCSER_TEMT;
 141
 142         return 0;
 143 }
 144
 145 static void stm32_usart_rs485_rts_enable(struct uart_port *port)
 146 {
 147         struct stm32_port *stm32_port = to_stm32_port(port);
 148         struct serial_rs485 *rs485conf = &port->rs485;
 149
 150         if (stm32_port->hw_flow_control ||
 151             !(rs485conf->flags & SER_RS485_ENABLED))
 152                 return;
 153
 154         if (rs485conf->flags & SER_RS485_RTS_ON_SEND) {
 155                 mctrl_gpio_set(stm32_port->gpios,
 156                                stm32_port->port.mctrl | TIOCM_RTS);
 157         } else {
 158                 mctrl_gpio_set(stm32_port->gpios,
 159                                stm32_port->port.mctrl & ~TIOCM_RTS);
 160         }
 161 }
 162
 163 static void stm32_usart_rs485_rts_disable(struct uart_port *port)
 164 {
 165         struct stm32_port *stm32_port = to_stm32_port(port);
 166         struct serial_rs485 *rs485conf = &port->rs485;
 167
 168         if (stm32_port->hw_flow_control ||
 169             !(rs485conf->flags & SER_RS485_ENABLED))
 170                 return;
 171
 172         if (rs485conf->flags & SER_RS485_RTS_ON_SEND) {
 173                 mctrl_gpio_set(stm32_port->gpios,
 174                                stm32_port->port.mctrl & ~TIOCM_RTS);
 175         } else {
 176                 mctrl_gpio_set(stm32_port->gpios,
 177                                stm32_port->port.mctrl | TIOCM_RTS);
 178         }
 179 }
 180
 181 static void stm32_usart_config_reg_rs485(u32 *cr1, u32 *cr3, u32 delay_ADE,
 182                                          u32 delay_DDE, u32 baud)
 183 {
 184         u32 rs485_deat_dedt;
 185         u32 rs485_deat_dedt_max = (USART_CR1_DEAT_MASK >> USART_CR1_DEAT_SHIFT);
 186         bool over8;
 187
 188         *cr3 |= USART_CR3_DEM;
 189         over8 = *cr1 & USART_CR1_OVER8;
 190
 191         *cr1 &= ~(USART_CR1_DEDT_MASK | USART_CR1_DEAT_MASK);
 192
 193         if (over8)
 194                 rs485_deat_dedt = delay_ADE * baud * 8;
 195         else
 196                 rs485_deat_dedt = delay_ADE * baud * 16;
 197
 198         rs485_deat_dedt = DIV_ROUND_CLOSEST(rs485_deat_dedt, 1000);
 199         rs485_deat_dedt = rs485_deat_dedt > rs485_deat_dedt_max ?
 200                           rs485_deat_dedt_max : rs485_deat_dedt;
 201         rs485_deat_dedt = (rs485_deat_dedt << USART_CR1_DEAT_SHIFT) &
 202                            USART_CR1_DEAT_MASK;
 203         *cr1 |= rs485_deat_dedt;
 204
 205         if (over8)
 206                 rs485_deat_dedt = delay_DDE * baud * 8;
 207         else
 208                 rs485_deat_dedt = delay_DDE * baud * 16;
 209
 210         rs485_deat_dedt = DIV_ROUND_CLOSEST(rs485_deat_dedt, 1000);
 211         rs485_deat_dedt = rs485_deat_dedt > rs485_deat_dedt_max ?
 212                           rs485_deat_dedt_max : rs485_deat_dedt;
 213         rs485_deat_dedt = (rs485_deat_dedt << USART_CR1_DEDT_SHIFT) &
 214                            USART_CR1_DEDT_MASK;
 215         *cr1 |= rs485_deat_dedt;
 216 }
 217
 218 static int stm32_usart_config_rs485(struct uart_port *port, struct ktermios *termios,
 219                                     struct serial_rs485 *rs485conf)
 220 {
 221         struct stm32_port *stm32_port = to_stm32_port(port);
 222         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 223         const struct stm32_usart_config *cfg = &stm32_port->info->cfg;
 224         u32 usartdiv, baud, cr1, cr3;
 225         bool over8;
 226
 227         stm32_usart_clr_bits(port, ofs->cr1, BIT(cfg->uart_enable_bit));
 228
 229         rs485conf->flags |= SER_RS485_RX_DURING_TX;
 230
 231         if (rs485conf->flags & SER_RS485_ENABLED) {
 232                 cr1 = readl_relaxed(port->membase + ofs->cr1);
 233                 cr3 = readl_relaxed(port->membase + ofs->cr3);
 234                 usartdiv = readl_relaxed(port->membase + ofs->brr);
 235                 usartdiv = usartdiv & GENMASK(15, 0);
 236                 over8 = cr1 & USART_CR1_OVER8;
 237
 238                 if (over8)
 239                         usartdiv = usartdiv | (usartdiv & GENMASK(4, 0))
 240                                    << USART_BRR_04_R_SHIFT;
 241
 242                 baud = DIV_ROUND_CLOSEST(port->uartclk, usartdiv);
 243                 stm32_usart_config_reg_rs485(&cr1, &cr3,
 244                                              rs485conf->delay_rts_before_send,
 245                                              rs485conf->delay_rts_after_send,
 246                                              baud);
 247
 248                 if (rs485conf->flags & SER_RS485_RTS_ON_SEND)
 249                         cr3 &= ~USART_CR3_DEP;
 250                 else
 251                         cr3 |= USART_CR3_DEP;
 252
 253                 writel_relaxed(cr3, port->membase + ofs->cr3);
 254                 writel_relaxed(cr1, port->membase + ofs->cr1);
 255         } else {
 256                 stm32_usart_clr_bits(port, ofs->cr3,
 257                                      USART_CR3_DEM | USART_CR3_DEP);
 258                 stm32_usart_clr_bits(port, ofs->cr1,
 259                                      USART_CR1_DEDT_MASK | USART_CR1_DEAT_MASK);
 260         }
 261
 262         stm32_usart_set_bits(port, ofs->cr1, BIT(cfg->uart_enable_bit));
 263
 264         /* Adjust RTS polarity in case it's driven in software */
 265         if (stm32_usart_tx_empty(port))
 266                 stm32_usart_rs485_rts_disable(port);
 267         else
 268                 stm32_usart_rs485_rts_enable(port);
 269
 270         return 0;
 271 }
 272
 273 static int stm32_usart_init_rs485(struct uart_port *port,
 274                                   struct platform_device *pdev)
 275 {
 276         struct serial_rs485 *rs485conf = &port->rs485;
 277
 278         rs485conf->flags = 0;
 279         rs485conf->delay_rts_before_send = 0;
 280         rs485conf->delay_rts_after_send = 0;
 281
 282         if (!pdev->dev.of_node)
 283                 return -ENODEV;
 284
 285         return uart_get_rs485_mode(port);
 286 }
 287
 288 static bool stm32_usart_rx_dma_enabled(struct uart_port *port)
 289 {
 290         struct stm32_port *stm32_port = to_stm32_port(port);
 291         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 292
 293         if (!stm32_port->rx_ch)
 294                 return false;
 295
 296         return !!(readl_relaxed(port->membase + ofs->cr3) & USART_CR3_DMAR);
 297 }
 298
 299 /* Return true when data is pending (in pio mode), and false when no data is pending. */
 300 static bool stm32_usart_pending_rx_pio(struct uart_port *port, u32 *sr)
 301 {
 302         struct stm32_port *stm32_port = to_stm32_port(port);
 303         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 304
 305         *sr = readl_relaxed(port->membase + ofs->isr);
 306         /* Get pending characters in RDR or FIFO */
 307         if (*sr & USART_SR_RXNE) {
 308                 /* Get all pending characters from the RDR or the FIFO when using interrupts */
 309                 if (!stm32_usart_rx_dma_enabled(port))
 310                         return true;
 311
 312                 /* Handle only RX data errors when using DMA */
 313                 if (*sr & USART_SR_ERR_MASK)
 314                         return true;
 315         }
 316
 317         return false;
 318 }
 319
 320 static unsigned long stm32_usart_get_char_pio(struct uart_port *port)
 321 {
 322         struct stm32_port *stm32_port = to_stm32_port(port);
 323         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 324         unsigned long c;
 325
 326         c = readl_relaxed(port->membase + ofs->rdr);
 327         /* Apply RDR data mask */
 328         c &= stm32_port->rdr_mask;
 329
 330         return c;
 331 }
 332
 333 static unsigned int stm32_usart_receive_chars_pio(struct uart_port *port)
 334 {
 335         struct stm32_port *stm32_port = to_stm32_port(port);
 336         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 337         unsigned long c;
 338         unsigned int size = 0;
 339         u32 sr;
 340         char flag;
 341
 342         while (stm32_usart_pending_rx_pio(port, &sr)) {
 343                 sr |= USART_SR_DUMMY_RX;
 344                 flag = TTY_NORMAL;
 345
 346                 /*
 347                  * Status bits has to be cleared before reading the RDR:
 348                  * In FIFO mode, reading the RDR will pop the next data
 349                  * (if any) along with its status bits into the SR.
 350                  * Not doing so leads to misalignement between RDR and SR,
 351                  * and clear status bits of the next rx data.
 352                  *
 353                  * Clear errors flags for stm32f7 and stm32h7 compatible
 354                  * devices. On stm32f4 compatible devices, the error bit is
 355                  * cleared by the sequence [read SR - read DR].
 356                  */
 357                 if ((sr & USART_SR_ERR_MASK) && ofs->icr != UNDEF_REG)
 358                         writel_relaxed(sr & USART_SR_ERR_MASK,
 359                                        port->membase + ofs->icr);
 360
 361                 c = stm32_usart_get_char_pio(port);
 362                 port->icount.rx++;
 363                 size++;
 364                 if (sr & USART_SR_ERR_MASK) {
 365                         if (sr & USART_SR_ORE) {
 366                                 port->icount.overrun++;
 367                         } else if (sr & USART_SR_PE) {
 368                                 port->icount.parity++;
 369                         } else if (sr & USART_SR_FE) {
 370                                 /* Break detection if character is null */
 371                                 if (!c) {
 372                                         port->icount.brk++;
 373                                         if (uart_handle_break(port))
 374                                                 continue;
 375                                 } else {
 376                                         port->icount.frame++;
 377                                 }
 378                         }
 379
 380                         sr &= port->read_status_mask;
 381
 382                         if (sr & USART_SR_PE) {
 383                                 flag = TTY_PARITY;
 384                         } else if (sr & USART_SR_FE) {
 385                                 if (!c)
 386                                         flag = TTY_BREAK;
 387                                 else
 388                                         flag = TTY_FRAME;
 389                         }
 390                 }
 391
 392                 if (uart_prepare_sysrq_char(port, c))
 393                         continue;
 394                 uart_insert_char(port, sr, USART_SR_ORE, c, flag);
 395         }
 396
 397         return size;
 398 }
 399
 400 static void stm32_usart_push_buffer_dma(struct uart_port *port, unsigned int dma_size)
 401 {
 402         struct stm32_port *stm32_port = to_stm32_port(port);
 403         struct tty_port *ttyport = &stm32_port->port.state->port;
 404         unsigned char *dma_start;
 405         int dma_count, i;
 406
 407         dma_start = stm32_port->rx_buf + (RX_BUF_L - stm32_port->last_res);
 408
 409         /*
 410          * Apply rdr_mask on buffer in order to mask parity bit.
 411          * This loop is useless in cs8 mode because DMA copies only
 412          * 8 bits and already ignores parity bit.
 413          */
 414         if (!(stm32_port->rdr_mask == (BIT(8) - 1)))
 415                 for (i = 0; i < dma_size; i++)
 416                         *(dma_start + i) &= stm32_port->rdr_mask;
 417
 418         dma_count = tty_insert_flip_string(ttyport, dma_start, dma_size);
 419         port->icount.rx += dma_count;
 420         if (dma_count != dma_size)
 421                 port->icount.buf_overrun++;
 422         stm32_port->last_res -= dma_count;
 423         if (stm32_port->last_res == 0)
 424                 stm32_port->last_res = RX_BUF_L;
 425 }
 426
 427 static unsigned int stm32_usart_receive_chars_dma(struct uart_port *port)
 428 {
 429         struct stm32_port *stm32_port = to_stm32_port(port);
 430         unsigned int dma_size, size = 0;
 431
 432         /* DMA buffer is configured in cyclic mode and handles the rollback of the buffer. */
 433         if (stm32_port->rx_dma_state.residue > stm32_port->last_res) {
 434                 /* Conditional first part: from last_res to end of DMA buffer */
 435                 dma_size = stm32_port->last_res;
 436                 stm32_usart_push_buffer_dma(port, dma_size);
 437                 size = dma_size;
 438         }
 439
 440         dma_size = stm32_port->last_res - stm32_port->rx_dma_state.residue;
 441         stm32_usart_push_buffer_dma(port, dma_size);
 442         size += dma_size;
 443
 444         return size;
 445 }
 446
 447 static unsigned int stm32_usart_receive_chars(struct uart_port *port, bool force_dma_flush)
 448 {
 449         struct stm32_port *stm32_port = to_stm32_port(port);
 450         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 451         enum dma_status rx_dma_status;
 452         u32 sr;
 453         unsigned int size = 0;
 454
 455         if (stm32_usart_rx_dma_enabled(port) || force_dma_flush) {
 456                 rx_dma_status = dmaengine_tx_status(stm32_port->rx_ch,
 457                                                     stm32_port->rx_ch->cookie,
 458                                                     &stm32_port->rx_dma_state);
 459                 if (rx_dma_status == DMA_IN_PROGRESS) {
 460                         /* Empty DMA buffer */
 461                         size = stm32_usart_receive_chars_dma(port);
 462                         sr = readl_relaxed(port->membase + ofs->isr);
 463                         if (sr & USART_SR_ERR_MASK) {
 464                                 /* Disable DMA request line */
 465                                 stm32_usart_clr_bits(port, ofs->cr3, USART_CR3_DMAR);
 466
 467                                 /* Switch to PIO mode to handle the errors */
 468                                 size += stm32_usart_receive_chars_pio(port);
 469
 470                                 /* Switch back to DMA mode */
 471                                 stm32_usart_set_bits(port, ofs->cr3, USART_CR3_DMAR);
 472                         }
 473                 } else {
 474                         /* Disable RX DMA */
 475                         dmaengine_terminate_async(stm32_port->rx_ch);
 476                         stm32_usart_clr_bits(port, ofs->cr3, USART_CR3_DMAR);
 477                         /* Fall back to interrupt mode */
 478                         dev_dbg(port->dev, "DMA error, fallback to irq mode\n");
 479                         size = stm32_usart_receive_chars_pio(port);
 480                 }
 481         } else {
 482                 size = stm32_usart_receive_chars_pio(port);
 483         }
 484
 485         return size;
 486 }
 487
 488 static void stm32_usart_tx_dma_terminate(struct stm32_port *stm32_port)
 489 {
 490         dmaengine_terminate_async(stm32_port->tx_ch);
 491         stm32_port->tx_dma_busy = false;
 492 }
 493
 494 static bool stm32_usart_tx_dma_started(struct stm32_port *stm32_port)
 495 {
 496         /*
 497          * We cannot use the function "dmaengine_tx_status" to know the
 498          * status of DMA. This function does not show if the "dma complete"
 499          * callback of the DMA transaction has been called. So we prefer
 500          * to use "tx_dma_busy" flag to prevent dual DMA transaction at the
 501          * same time.
 502          */
 503         return stm32_port->tx_dma_busy;
 504 }
 505
 506 static bool stm32_usart_tx_dma_enabled(struct stm32_port *stm32_port)
 507 {
 508         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 509
 510         return !!(readl_relaxed(stm32_port->port.membase + ofs->cr3) & USART_CR3_DMAT);
 511 }
 512
 513 static void stm32_usart_tx_dma_complete(void *arg)
 514 {
 515         struct uart_port *port = arg;
 516         struct stm32_port *stm32port = to_stm32_port(port);
 517         const struct stm32_usart_offsets *ofs = &stm32port->info->ofs;
 518         unsigned long flags;
 519
 520         stm32_usart_clr_bits(port, ofs->cr3, USART_CR3_DMAT);
 521         stm32_usart_tx_dma_terminate(stm32port);
 522
 523         /* Let's see if we have pending data to send */
 524         spin_lock_irqsave(&port->lock, flags);
 525         stm32_usart_transmit_chars(port);
 526         spin_unlock_irqrestore(&port->lock, flags);
 527 }
 528
 529 static void stm32_usart_tx_interrupt_enable(struct uart_port *port)
 530 {
 531         struct stm32_port *stm32_port = to_stm32_port(port);
 532         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 533
 534         /*
 535          * Enables TX FIFO threashold irq when FIFO is enabled,
 536          * or TX empty irq when FIFO is disabled
 537          */
 538         if (stm32_port->fifoen && stm32_port->txftcfg >= 0)
 539                 stm32_usart_set_bits(port, ofs->cr3, USART_CR3_TXFTIE);
 540         else
 541                 stm32_usart_set_bits(port, ofs->cr1, USART_CR1_TXEIE);
 542 }
 543
 544 static void stm32_usart_tc_interrupt_enable(struct uart_port *port)
 545 {
 546         struct stm32_port *stm32_port = to_stm32_port(port);
 547         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 548
 549         stm32_usart_set_bits(port, ofs->cr1, USART_CR1_TCIE);
 550 }
 551
 552 static void stm32_usart_rx_dma_complete(void *arg)
 553 {
 554         struct uart_port *port = arg;
 555         struct tty_port *tport = &port->state->port;
 556         unsigned int size;
 557         unsigned long flags;
 558
 559         spin_lock_irqsave(&port->lock, flags);
 560         size = stm32_usart_receive_chars(port, false);
 561         uart_unlock_and_check_sysrq_irqrestore(port, flags);
 562         if (size)
 563                 tty_flip_buffer_push(tport);
 564 }
 565
 566 static void stm32_usart_tx_interrupt_disable(struct uart_port *port)
 567 {
 568         struct stm32_port *stm32_port = to_stm32_port(port);
 569         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 570
 571         if (stm32_port->fifoen && stm32_port->txftcfg >= 0)
 572                 stm32_usart_clr_bits(port, ofs->cr3, USART_CR3_TXFTIE);
 573         else
 574                 stm32_usart_clr_bits(port, ofs->cr1, USART_CR1_TXEIE);
 575 }
 576
 577 static void stm32_usart_tc_interrupt_disable(struct uart_port *port)
 578 {
 579         struct stm32_port *stm32_port = to_stm32_port(port);
 580         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 581
 582         stm32_usart_clr_bits(port, ofs->cr1, USART_CR1_TCIE);
 583 }
 584
 585 static void stm32_usart_transmit_chars_pio(struct uart_port *port)
 586 {
 587         struct stm32_port *stm32_port = to_stm32_port(port);
 588         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 589         struct circ_buf *xmit = &port->state->xmit;
 590
 591         if (stm32_usart_tx_dma_enabled(stm32_port))
 592                 stm32_usart_clr_bits(port, ofs->cr3, USART_CR3_DMAT);
 593
 594         while (!uart_circ_empty(xmit)) {
 595                 /* Check that TDR is empty before filling FIFO */
 596                 if (!(readl_relaxed(port->membase + ofs->isr) & USART_SR_TXE))
 597                         break;
 598                 writel_relaxed(xmit->buf[xmit->tail], port->membase + ofs->tdr);
 599                 xmit->tail = (xmit->tail + 1) & (UART_XMIT_SIZE - 1);
 600                 port->icount.tx++;
 601         }
 602
 603         /* rely on TXE irq (mask or unmask) for sending remaining data */
 604         if (uart_circ_empty(xmit))
 605                 stm32_usart_tx_interrupt_disable(port);
 606         else
 607                 stm32_usart_tx_interrupt_enable(port);
 608 }
 609
 610 static void stm32_usart_transmit_chars_dma(struct uart_port *port)
 611 {
 612         struct stm32_port *stm32port = to_stm32_port(port);
 613         const struct stm32_usart_offsets *ofs = &stm32port->info->ofs;
 614         struct circ_buf *xmit = &port->state->xmit;
 615         struct dma_async_tx_descriptor *desc = NULL;
 616         unsigned int count;
 617
 618         if (stm32_usart_tx_dma_started(stm32port)) {
 619                 if (!stm32_usart_tx_dma_enabled(stm32port))
 620                         stm32_usart_set_bits(port, ofs->cr3, USART_CR3_DMAT);
 621                 return;
 622         }
 623
 624         count = uart_circ_chars_pending(xmit);
 625
 626         if (count > TX_BUF_L)
 627                 count = TX_BUF_L;
 628
 629         if (xmit->tail < xmit->head) {
 630                 memcpy(&stm32port->tx_buf[0], &xmit->buf[xmit->tail], count);
 631         } else {
 632                 size_t one = UART_XMIT_SIZE - xmit->tail;
 633                 size_t two;
 634
 635                 if (one > count)
 636                         one = count;
 637                 two = count - one;
 638
 639                 memcpy(&stm32port->tx_buf[0], &xmit->buf[xmit->tail], one);
 640                 if (two)
 641                         memcpy(&stm32port->tx_buf[one], &xmit->buf[0], two);
 642         }
 643
 644         desc = dmaengine_prep_slave_single(stm32port->tx_ch,
 645                                            stm32port->tx_dma_buf,
 646                                            count,
 647                                            DMA_MEM_TO_DEV,
 648                                            DMA_PREP_INTERRUPT);
 649
 650         if (!desc)
 651                 goto fallback_err;
 652
 653         /*
 654          * Set "tx_dma_busy" flag. This flag will be released when
 655          * dmaengine_terminate_async will be called. This flag helps
 656          * transmit_chars_dma not to start another DMA transaction
 657          * if the callback of the previous is not yet called.
 658          */
 659         stm32port->tx_dma_busy = true;
 660
 661         desc->callback = stm32_usart_tx_dma_complete;
 662         desc->callback_param = port;
 663
 664         /* Push current DMA TX transaction in the pending queue */
 665         if (dma_submit_error(dmaengine_submit(desc))) {
 666                 /* dma no yet started, safe to free resources */
 667                 stm32_usart_tx_dma_terminate(stm32port);
 668                 goto fallback_err;
 669         }
 670
 671         /* Issue pending DMA TX requests */
 672         dma_async_issue_pending(stm32port->tx_ch);
 673
 674         stm32_usart_set_bits(port, ofs->cr3, USART_CR3_DMAT);
 675
 676         xmit->tail = (xmit->tail + count) & (UART_XMIT_SIZE - 1);
 677         port->icount.tx += count;
 678         return;
 679
 680 fallback_err:
 681         stm32_usart_transmit_chars_pio(port);
 682 }
 683
 684 static void stm32_usart_transmit_chars(struct uart_port *port)
 685 {
 686         struct stm32_port *stm32_port = to_stm32_port(port);
 687         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 688         struct circ_buf *xmit = &port->state->xmit;
 689         u32 isr;
 690         int ret;
 691
 692         if (!stm32_port->hw_flow_control &&
 693             port->rs485.flags & SER_RS485_ENABLED) {
 694                 stm32_port->txdone = false;
 695                 stm32_usart_tc_interrupt_disable(port);
 696                 stm32_usart_rs485_rts_enable(port);
 697         }
 698
 699         if (port->x_char) {
 700                 if (stm32_usart_tx_dma_started(stm32_port) &&
 701                     stm32_usart_tx_dma_enabled(stm32_port))
 702                         stm32_usart_clr_bits(port, ofs->cr3, USART_CR3_DMAT);
 703
 704                 /* Check that TDR is empty before filling FIFO */
 705                 ret =
 706                 readl_relaxed_poll_timeout_atomic(port->membase + ofs->isr,
 707                                                   isr,
 708                                                   (isr & USART_SR_TXE),
 709                                                   10, 1000);
 710                 if (ret)
 711                         dev_warn(port->dev, "1 character may be erased\n");
 712
 713                 writel_relaxed(port->x_char, port->membase + ofs->tdr);
 714                 port->x_char = 0;
 715                 port->icount.tx++;
 716                 if (stm32_usart_tx_dma_started(stm32_port))
 717                         stm32_usart_set_bits(port, ofs->cr3, USART_CR3_DMAT);
 718                 return;
 719         }
 720
 721         if (uart_circ_empty(xmit) || uart_tx_stopped(port)) {
 722                 stm32_usart_tx_interrupt_disable(port);
 723                 return;
 724         }
 725
 726         if (ofs->icr == UNDEF_REG)
 727                 stm32_usart_clr_bits(port, ofs->isr, USART_SR_TC);
 728         else
 729                 writel_relaxed(USART_ICR_TCCF, port->membase + ofs->icr);
 730
 731         if (stm32_port->tx_ch)
 732                 stm32_usart_transmit_chars_dma(port);
 733         else
 734                 stm32_usart_transmit_chars_pio(port);
 735
 736         if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
 737                 uart_write_wakeup(port);
 738
 739         if (uart_circ_empty(xmit)) {
 740                 stm32_usart_tx_interrupt_disable(port);
 741                 if (!stm32_port->hw_flow_control &&
 742                     port->rs485.flags & SER_RS485_ENABLED) {
 743                         stm32_port->txdone = true;
 744                         stm32_usart_tc_interrupt_enable(port);
 745                 }
 746         }
 747 }
 748
 749 static irqreturn_t stm32_usart_interrupt(int irq, void *ptr)
 750 {
 751         struct uart_port *port = ptr;
 752         struct tty_port *tport = &port->state->port;
 753         struct stm32_port *stm32_port = to_stm32_port(port);
 754         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 755         u32 sr;
 756         unsigned int size;
 757
 758         sr = readl_relaxed(port->membase + ofs->isr);
 759
 760         if (!stm32_port->hw_flow_control &&
 761             port->rs485.flags & SER_RS485_ENABLED &&
 762             (sr & USART_SR_TC)) {
 763                 stm32_usart_tc_interrupt_disable(port);
 764                 stm32_usart_rs485_rts_disable(port);
 765         }
 766
 767         if ((sr & USART_SR_RTOF) && ofs->icr != UNDEF_REG)
 768                 writel_relaxed(USART_ICR_RTOCF,
 769                                port->membase + ofs->icr);
 770
 771         if ((sr & USART_SR_WUF) && ofs->icr != UNDEF_REG) {
 772                 /* Clear wake up flag and disable wake up interrupt */
 773                 writel_relaxed(USART_ICR_WUCF,
 774                                port->membase + ofs->icr);
 775                 stm32_usart_clr_bits(port, ofs->cr3, USART_CR3_WUFIE);
 776                 if (irqd_is_wakeup_set(irq_get_irq_data(port->irq)))
 777                         pm_wakeup_event(tport->tty->dev, 0);
 778         }
 779
 780         /*
 781          * rx errors in dma mode has to be handled ASAP to avoid overrun as the DMA request
 782          * line has been masked by HW and rx data are stacking in FIFO.
 783          */
 784         if (!stm32_port->throttled) {
 785                 if (((sr & USART_SR_RXNE) && !stm32_usart_rx_dma_enabled(port)) ||
 786                     ((sr & USART_SR_ERR_MASK) && stm32_usart_rx_dma_enabled(port))) {
 787                         spin_lock(&port->lock);
 788                         size = stm32_usart_receive_chars(port, false);
 789                         uart_unlock_and_check_sysrq(port);
 790                         if (size)
 791                                 tty_flip_buffer_push(tport);
 792                 }
 793         }
 794
 795         if ((sr & USART_SR_TXE) && !(stm32_port->tx_ch)) {
 796                 spin_lock(&port->lock);
 797                 stm32_usart_transmit_chars(port);
 798                 spin_unlock(&port->lock);
 799         }
 800
 801         /* Receiver timeout irq for DMA RX */
 802         if (stm32_usart_rx_dma_enabled(port) && !stm32_port->throttled) {
 803                 spin_lock(&port->lock);
 804                 size = stm32_usart_receive_chars(port, false);
 805                 uart_unlock_and_check_sysrq(port);
 806                 if (size)
 807                         tty_flip_buffer_push(tport);
 808         }
 809
 810         return IRQ_HANDLED;
 811 }
 812
 813 static void stm32_usart_set_mctrl(struct uart_port *port, unsigned int mctrl)
 814 {
 815         struct stm32_port *stm32_port = to_stm32_port(port);
 816         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 817
 818         if ((mctrl & TIOCM_RTS) && (port->status & UPSTAT_AUTORTS))
 819                 stm32_usart_set_bits(port, ofs->cr3, USART_CR3_RTSE);
 820         else
 821                 stm32_usart_clr_bits(port, ofs->cr3, USART_CR3_RTSE);
 822
 823         mctrl_gpio_set(stm32_port->gpios, mctrl);
 824 }
 825
 826 static unsigned int stm32_usart_get_mctrl(struct uart_port *port)
 827 {
 828         struct stm32_port *stm32_port = to_stm32_port(port);
 829         unsigned int ret;
 830
 831         /* This routine is used to get signals of: DCD, DSR, RI, and CTS */
 832         ret = TIOCM_CAR | TIOCM_DSR | TIOCM_CTS;
 833
 834         return mctrl_gpio_get(stm32_port->gpios, &ret);
 835 }
 836
 837 static void stm32_usart_enable_ms(struct uart_port *port)
 838 {
 839         mctrl_gpio_enable_ms(to_stm32_port(port)->gpios);
 840 }
 841
 842 static void stm32_usart_disable_ms(struct uart_port *port)
 843 {
 844         mctrl_gpio_disable_ms(to_stm32_port(port)->gpios);
 845 }
 846
 847 /* Transmit stop */
 848 static void stm32_usart_stop_tx(struct uart_port *port)
 849 {
 850         struct stm32_port *stm32_port = to_stm32_port(port);
 851         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 852
 853         stm32_usart_tx_interrupt_disable(port);
 854         if (stm32_usart_tx_dma_started(stm32_port) && stm32_usart_tx_dma_enabled(stm32_port))
 855                 stm32_usart_clr_bits(port, ofs->cr3, USART_CR3_DMAT);
 856
 857         stm32_usart_rs485_rts_disable(port);
 858 }
 859
 860 /* There are probably characters waiting to be transmitted. */
 861 static void stm32_usart_start_tx(struct uart_port *port)
 862 {
 863         struct circ_buf *xmit = &port->state->xmit;
 864
 865         if (uart_circ_empty(xmit) && !port->x_char) {
 866                 stm32_usart_rs485_rts_disable(port);
 867                 return;
 868         }
 869
 870         stm32_usart_rs485_rts_enable(port);
 871
 872         stm32_usart_transmit_chars(port);
 873 }
 874
 875 /* Flush the transmit buffer. */
 876 static void stm32_usart_flush_buffer(struct uart_port *port)
 877 {
 878         struct stm32_port *stm32_port = to_stm32_port(port);
 879         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 880
 881         if (stm32_port->tx_ch) {
 882                 stm32_usart_tx_dma_terminate(stm32_port);
 883                 stm32_usart_clr_bits(port, ofs->cr3, USART_CR3_DMAT);
 884         }
 885 }
 886
 887 /* Throttle the remote when input buffer is about to overflow. */
 888 static void stm32_usart_throttle(struct uart_port *port)
 889 {
 890         struct stm32_port *stm32_port = to_stm32_port(port);
 891         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 892         unsigned long flags;
 893
 894         spin_lock_irqsave(&port->lock, flags);
 895
 896         /*
 897          * Disable DMA request line if enabled, so the RX data gets queued into the FIFO.
 898          * Hardware flow control is triggered when RX FIFO is full.
 899          */
 900         if (stm32_usart_rx_dma_enabled(port))
 901                 stm32_usart_clr_bits(port, ofs->cr3, USART_CR3_DMAR);
 902
 903         stm32_usart_clr_bits(port, ofs->cr1, stm32_port->cr1_irq);
 904         if (stm32_port->cr3_irq)
 905                 stm32_usart_clr_bits(port, ofs->cr3, stm32_port->cr3_irq);
 906
 907         stm32_port->throttled = true;
 908         spin_unlock_irqrestore(&port->lock, flags);
 909 }
 910
 911 /* Unthrottle the remote, the input buffer can now accept data. */
 912 static void stm32_usart_unthrottle(struct uart_port *port)
 913 {
 914         struct stm32_port *stm32_port = to_stm32_port(port);
 915         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 916         unsigned long flags;
 917
 918         spin_lock_irqsave(&port->lock, flags);
 919         stm32_usart_set_bits(port, ofs->cr1, stm32_port->cr1_irq);
 920         if (stm32_port->cr3_irq)
 921                 stm32_usart_set_bits(port, ofs->cr3, stm32_port->cr3_irq);
 922
 923         /*
 924          * Switch back to DMA mode (re-enable DMA request line).
 925          * Hardware flow control is stopped when FIFO is not full any more.
 926          */
 927         if (stm32_port->rx_ch)
 928                 stm32_usart_set_bits(port, ofs->cr3, USART_CR3_DMAR);
 929
 930         stm32_port->throttled = false;
 931         spin_unlock_irqrestore(&port->lock, flags);
 932 }
 933
 934 /* Receive stop */
 935 static void stm32_usart_stop_rx(struct uart_port *port)
 936 {
 937         struct stm32_port *stm32_port = to_stm32_port(port);
 938         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 939
 940         /* Disable DMA request line. */
 941         if (stm32_port->rx_ch)
 942                 stm32_usart_clr_bits(port, ofs->cr3, USART_CR3_DMAR);
 943
 944         stm32_usart_clr_bits(port, ofs->cr1, stm32_port->cr1_irq);
 945         if (stm32_port->cr3_irq)
 946                 stm32_usart_clr_bits(port, ofs->cr3, stm32_port->cr3_irq);
 947 }
 948
 949 /* Handle breaks - ignored by us */
 950 static void stm32_usart_break_ctl(struct uart_port *port, int break_state)
 951 {
 952 }
 953
 954 static int stm32_usart_start_rx_dma_cyclic(struct uart_port *port)
 955 {
 956         struct stm32_port *stm32_port = to_stm32_port(port);
 957         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
 958         struct dma_async_tx_descriptor *desc;
 959         int ret;
 960
 961         stm32_port->last_res = RX_BUF_L;
 962         /* Prepare a DMA cyclic transaction */
 963         desc = dmaengine_prep_dma_cyclic(stm32_port->rx_ch,
 964                                          stm32_port->rx_dma_buf,
 965                                          RX_BUF_L, RX_BUF_P,
 966                                          DMA_DEV_TO_MEM,
 967                                          DMA_PREP_INTERRUPT);
 968         if (!desc) {
 969                 dev_err(port->dev, "rx dma prep cyclic failed\n");
 970                 return -ENODEV;
 971         }
 972
 973         desc->callback = stm32_usart_rx_dma_complete;
 974         desc->callback_param = port;
 975
 976         /* Push current DMA transaction in the pending queue */
 977         ret = dma_submit_error(dmaengine_submit(desc));
 978         if (ret) {
 979                 dmaengine_terminate_sync(stm32_port->rx_ch);
 980                 return ret;
 981         }
 982
 983         /* Issue pending DMA requests */
 984         dma_async_issue_pending(stm32_port->rx_ch);
 985
 986         /*
 987          * DMA request line not re-enabled at resume when port is throttled.
 988          * It will be re-enabled by unthrottle ops.
 989          */
 990         if (!stm32_port->throttled)
 991                 stm32_usart_set_bits(port, ofs->cr3, USART_CR3_DMAR);
 992
 993         return 0;
 994 }
 995
 996 static int stm32_usart_startup(struct uart_port *port)
 997 {
 998         struct stm32_port *stm32_port = to_stm32_port(port);
 999         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
1000         const struct stm32_usart_config *cfg = &stm32_port->info->cfg;
1001         const char *name = to_platform_device(port->dev)->name;
1002         u32 val;
1003         int ret;
1004
1005         ret = request_irq(port->irq, stm32_usart_interrupt,
1006                           IRQF_NO_SUSPEND, name, port);
1007         if (ret)
1008                 return ret;
1009
1010         if (stm32_port->swap) {
1011                 val = readl_relaxed(port->membase + ofs->cr2);
1012                 val |= USART_CR2_SWAP;
1013                 writel_relaxed(val, port->membase + ofs->cr2);
1014         }
1015
1016         /* RX FIFO Flush */
1017         if (ofs->rqr != UNDEF_REG)
1018                 writel_relaxed(USART_RQR_RXFRQ, port->membase + ofs->rqr);
1019
1020         if (stm32_port->rx_ch) {
1021                 ret = stm32_usart_start_rx_dma_cyclic(port);
1022                 if (ret) {
1023                         free_irq(port->irq, port);
1024                         return ret;
1025                 }
1026         }
1027
1028         /* RX enabling */
1029         val = stm32_port->cr1_irq | USART_CR1_RE | BIT(cfg->uart_enable_bit);
1030         stm32_usart_set_bits(port, ofs->cr1, val);
1031
1032         return 0;
1033 }
1034
1035 static void stm32_usart_shutdown(struct uart_port *port)
1036 {
1037         struct stm32_port *stm32_port = to_stm32_port(port);
1038         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
1039         const struct stm32_usart_config *cfg = &stm32_port->info->cfg;
1040         u32 val, isr;
1041         int ret;
1042
1043         if (stm32_usart_tx_dma_enabled(stm32_port))
1044                 stm32_usart_clr_bits(port, ofs->cr3, USART_CR3_DMAT);
1045
1046         if (stm32_usart_tx_dma_started(stm32_port))
1047                 stm32_usart_tx_dma_terminate(stm32_port);
1048
1049         /* Disable modem control interrupts */
1050         stm32_usart_disable_ms(port);
1051
1052         val = USART_CR1_TXEIE | USART_CR1_TE;
1053         val |= stm32_port->cr1_irq | USART_CR1_RE;
1054         val |= BIT(cfg->uart_enable_bit);
1055         if (stm32_port->fifoen)
1056                 val |= USART_CR1_FIFOEN;
1057
1058         ret = readl_relaxed_poll_timeout(port->membase + ofs->isr,
1059                                          isr, (isr & USART_SR_TC),
1060                                          10, 100000);
1061
1062         /* Send the TC error message only when ISR_TC is not set */
1063         if (ret)
1064                 dev_err(port->dev, "Transmission is not complete\n");
1065
1066         /* Disable RX DMA. */
1067         if (stm32_port->rx_ch)
1068                 dmaengine_terminate_async(stm32_port->rx_ch);
1069
1070         /* flush RX & TX FIFO */
1071         if (ofs->rqr != UNDEF_REG)
1072                 writel_relaxed(USART_RQR_TXFRQ | USART_RQR_RXFRQ,
1073                                port->membase + ofs->rqr);
1074
1075         stm32_usart_clr_bits(port, ofs->cr1, val);
1076
1077         free_irq(port->irq, port);
1078 }
1079
1080 static void stm32_usart_set_termios(struct uart_port *port,
1081                                     struct ktermios *termios,
1082                                     const struct ktermios *old)
1083 {
1084         struct stm32_port *stm32_port = to_stm32_port(port);
1085         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
1086         const struct stm32_usart_config *cfg = &stm32_port->info->cfg;
1087         struct serial_rs485 *rs485conf = &port->rs485;
1088         unsigned int baud, bits;
1089         u32 usartdiv, mantissa, fraction, oversampling;
1090         tcflag_t cflag = termios->c_cflag;
1091         u32 cr1, cr2, cr3, isr;
1092         unsigned long flags;
1093         int ret;
1094
1095         if (!stm32_port->hw_flow_control)
1096                 cflag &= ~CRTSCTS;
1097
1098         baud = uart_get_baud_rate(port, termios, old, 0, port->uartclk / 8);
1099
1100         spin_lock_irqsave(&port->lock, flags);
1101
1102         ret = readl_relaxed_poll_timeout_atomic(port->membase + ofs->isr,
1103                                                 isr,
1104                                                 (isr & USART_SR_TC),
1105                                                 10, 100000);
1106
1107         /* Send the TC error message only when ISR_TC is not set. */
1108         if (ret)
1109                 dev_err(port->dev, "Transmission is not complete\n");
1110
1111         /* Stop serial port and reset value */
1112         writel_relaxed(0, port->membase + ofs->cr1);
1113
1114         /* flush RX & TX FIFO */
1115         if (ofs->rqr != UNDEF_REG)
1116                 writel_relaxed(USART_RQR_TXFRQ | USART_RQR_RXFRQ,
1117                                port->membase + ofs->rqr);
1118
1119         cr1 = USART_CR1_TE | USART_CR1_RE;
1120         if (stm32_port->fifoen)
1121                 cr1 |= USART_CR1_FIFOEN;
1122         cr2 = stm32_port->swap ? USART_CR2_SWAP : 0;
1123
1124         /* Tx and RX FIFO configuration */
1125         cr3 = readl_relaxed(port->membase + ofs->cr3);
1126         cr3 &= USART_CR3_TXFTIE | USART_CR3_RXFTIE;
1127         if (stm32_port->fifoen) {
1128                 if (stm32_port->txftcfg >= 0)
1129                         cr3 |= stm32_port->txftcfg << USART_CR3_TXFTCFG_SHIFT;
1130                 if (stm32_port->rxftcfg >= 0)
1131                         cr3 |= stm32_port->rxftcfg << USART_CR3_RXFTCFG_SHIFT;
1132         }
1133
1134         if (cflag & CSTOPB)
1135                 cr2 |= USART_CR2_STOP_2B;
1136
1137         bits = tty_get_char_size(cflag);
1138         stm32_port->rdr_mask = (BIT(bits) - 1);
1139
1140         if (cflag & PARENB) {
1141                 bits++;
1142                 cr1 |= USART_CR1_PCE;
1143         }
1144
1145         /*
1146          * Word length configuration:
1147          * CS8 + parity, 9 bits word aka [M1:M0] = 0b01
1148          * CS7 or (CS6 + parity), 7 bits word aka [M1:M0] = 0b10
1149          * CS8 or (CS7 + parity), 8 bits word aka [M1:M0] = 0b00
1150          * M0 and M1 already cleared by cr1 initialization.
1151          */
1152         if (bits == 9) {
1153                 cr1 |= USART_CR1_M0;
1154         } else if ((bits == 7) && cfg->has_7bits_data) {
1155                 cr1 |= USART_CR1_M1;
1156         } else if (bits != 8) {
1157                 dev_dbg(port->dev, "Unsupported data bits config: %u bits\n"
1158                         , bits);
1159                 cflag &= ~CSIZE;
1160                 cflag |= CS8;
1161                 termios->c_cflag = cflag;
1162                 bits = 8;
1163                 if (cflag & PARENB) {
1164                         bits++;
1165                         cr1 |= USART_CR1_M0;
1166                 }
1167         }
1168
1169         if (ofs->rtor != UNDEF_REG && (stm32_port->rx_ch ||
1170                                        (stm32_port->fifoen &&
1171                                         stm32_port->rxftcfg >= 0))) {
1172                 if (cflag & CSTOPB)
1173                         bits = bits + 3; /* 1 start bit + 2 stop bits */
1174                 else
1175                         bits = bits + 2; /* 1 start bit + 1 stop bit */
1176
1177                 /* RX timeout irq to occur after last stop bit + bits */
1178                 stm32_port->cr1_irq = USART_CR1_RTOIE;
1179                 writel_relaxed(bits, port->membase + ofs->rtor);
1180                 cr2 |= USART_CR2_RTOEN;
1181                 /*
1182                  * Enable fifo threshold irq in two cases, either when there is no DMA, or when
1183                  * wake up over usart, from low power until the DMA gets re-enabled by resume.
1184                  */
1185                 stm32_port->cr3_irq =  USART_CR3_RXFTIE;
1186         }
1187
1188         cr1 |= stm32_port->cr1_irq;
1189         cr3 |= stm32_port->cr3_irq;
1190
1191         if (cflag & PARODD)
1192                 cr1 |= USART_CR1_PS;
1193
1194         port->status &= ~(UPSTAT_AUTOCTS | UPSTAT_AUTORTS);
1195         if (cflag & CRTSCTS) {
1196                 port->status |= UPSTAT_AUTOCTS | UPSTAT_AUTORTS;
1197                 cr3 |= USART_CR3_CTSE | USART_CR3_RTSE;
1198         }
1199
1200         usartdiv = DIV_ROUND_CLOSEST(port->uartclk, baud);
1201
1202         /*
1203          * The USART supports 16 or 8 times oversampling.
1204          * By default we prefer 16 times oversampling, so that the receiver
1205          * has a better tolerance to clock deviations.
1206          * 8 times oversampling is only used to achieve higher speeds.
1207          */
1208         if (usartdiv < 16) {
1209                 oversampling = 8;
1210                 cr1 |= USART_CR1_OVER8;
1211                 stm32_usart_set_bits(port, ofs->cr1, USART_CR1_OVER8);
1212         } else {
1213                 oversampling = 16;
1214                 cr1 &= ~USART_CR1_OVER8;
1215                 stm32_usart_clr_bits(port, ofs->cr1, USART_CR1_OVER8);
1216         }
1217
1218         mantissa = (usartdiv / oversampling) << USART_BRR_DIV_M_SHIFT;
1219         fraction = usartdiv % oversampling;
1220         writel_relaxed(mantissa | fraction, port->membase + ofs->brr);
1221
1222         uart_update_timeout(port, cflag, baud);
1223
1224         port->read_status_mask = USART_SR_ORE;
1225         if (termios->c_iflag & INPCK)
1226                 port->read_status_mask |= USART_SR_PE | USART_SR_FE;
1227         if (termios->c_iflag & (IGNBRK | BRKINT | PARMRK))
1228                 port->read_status_mask |= USART_SR_FE;
1229
1230         /* Characters to ignore */
1231         port->ignore_status_mask = 0;
1232         if (termios->c_iflag & IGNPAR)
1233                 port->ignore_status_mask = USART_SR_PE | USART_SR_FE;
1234         if (termios->c_iflag & IGNBRK) {
1235                 port->ignore_status_mask |= USART_SR_FE;
1236                 /*
1237                  * If we're ignoring parity and break indicators,
1238                  * ignore overruns too (for real raw support).
1239                  */
1240                 if (termios->c_iflag & IGNPAR)
1241                         port->ignore_status_mask |= USART_SR_ORE;
1242         }
1243
1244         /* Ignore all characters if CREAD is not set */
1245         if ((termios->c_cflag & CREAD) == 0)
1246                 port->ignore_status_mask |= USART_SR_DUMMY_RX;
1247
1248         if (stm32_port->rx_ch) {
1249                 /*
1250                  * Setup DMA to collect only valid data and enable error irqs.
1251                  * This also enables break reception when using DMA.
1252                  */
1253                 cr1 |= USART_CR1_PEIE;
1254                 cr3 |= USART_CR3_EIE;
1255                 cr3 |= USART_CR3_DMAR;
1256                 cr3 |= USART_CR3_DDRE;
1257         }
1258
1259         if (rs485conf->flags & SER_RS485_ENABLED) {
1260                 stm32_usart_config_reg_rs485(&cr1, &cr3,
1261                                              rs485conf->delay_rts_before_send,
1262                                              rs485conf->delay_rts_after_send,
1263                                              baud);
1264                 if (rs485conf->flags & SER_RS485_RTS_ON_SEND) {
1265                         cr3 &= ~USART_CR3_DEP;
1266                         rs485conf->flags &= ~SER_RS485_RTS_AFTER_SEND;
1267                 } else {
1268                         cr3 |= USART_CR3_DEP;
1269                         rs485conf->flags |= SER_RS485_RTS_AFTER_SEND;
1270                 }
1271
1272         } else {
1273                 cr3 &= ~(USART_CR3_DEM | USART_CR3_DEP);
1274                 cr1 &= ~(USART_CR1_DEDT_MASK | USART_CR1_DEAT_MASK);
1275         }
1276
1277         /* Configure wake up from low power on start bit detection */
1278         if (stm32_port->wakeup_src) {
1279                 cr3 &= ~USART_CR3_WUS_MASK;
1280                 cr3 |= USART_CR3_WUS_START_BIT;
1281         }
1282
1283         writel_relaxed(cr3, port->membase + ofs->cr3);
1284         writel_relaxed(cr2, port->membase + ofs->cr2);
1285         writel_relaxed(cr1, port->membase + ofs->cr1);
1286
1287         stm32_usart_set_bits(port, ofs->cr1, BIT(cfg->uart_enable_bit));
1288         spin_unlock_irqrestore(&port->lock, flags);
1289
1290         /* Handle modem control interrupts */
1291         if (UART_ENABLE_MS(port, termios->c_cflag))
1292                 stm32_usart_enable_ms(port);
1293         else
1294                 stm32_usart_disable_ms(port);
1295 }
1296
1297 static const char *stm32_usart_type(struct uart_port *port)
1298 {
1299         return (port->type == PORT_STM32) ? DRIVER_NAME : NULL;
1300 }
1301
1302 static void stm32_usart_release_port(struct uart_port *port)
1303 {
1304 }
1305
1306 static int stm32_usart_request_port(struct uart_port *port)
1307 {
1308         return 0;
1309 }
1310
1311 static void stm32_usart_config_port(struct uart_port *port, int flags)
1312 {
1313         if (flags & UART_CONFIG_TYPE)
1314                 port->type = PORT_STM32;
1315 }
1316
1317 static int
1318 stm32_usart_verify_port(struct uart_port *port, struct serial_struct *ser)
1319 {
1320         /* No user changeable parameters */
1321         return -EINVAL;
1322 }
1323
1324 static void stm32_usart_pm(struct uart_port *port, unsigned int state,
1325                            unsigned int oldstate)
1326 {
1327         struct stm32_port *stm32port = container_of(port,
1328                         struct stm32_port, port);
1329         const struct stm32_usart_offsets *ofs = &stm32port->info->ofs;
1330         const struct stm32_usart_config *cfg = &stm32port->info->cfg;
1331         unsigned long flags;
1332
1333         switch (state) {
1334         case UART_PM_STATE_ON:
1335                 pm_runtime_get_sync(port->dev);
1336                 break;
1337         case UART_PM_STATE_OFF:
1338                 spin_lock_irqsave(&port->lock, flags);
1339                 stm32_usart_clr_bits(port, ofs->cr1, BIT(cfg->uart_enable_bit));
1340                 spin_unlock_irqrestore(&port->lock, flags);
1341                 pm_runtime_put_sync(port->dev);
1342                 break;
1343         }
1344 }
1345
1346 #if defined(CONFIG_CONSOLE_POLL)
1347
1348  /* Callbacks for characters polling in debug context (i.e. KGDB). */
1349 static int stm32_usart_poll_init(struct uart_port *port)
1350 {
1351         struct stm32_port *stm32_port = to_stm32_port(port);
1352
1353         return clk_prepare_enable(stm32_port->clk);
1354 }
1355
1356 static int stm32_usart_poll_get_char(struct uart_port *port)
1357 {
1358         struct stm32_port *stm32_port = to_stm32_port(port);
1359         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
1360
1361         if (!(readl_relaxed(port->membase + ofs->isr) & USART_SR_RXNE))
1362                 return NO_POLL_CHAR;
1363
1364         return readl_relaxed(port->membase + ofs->rdr) & stm32_port->rdr_mask;
1365 }
1366
1367 static void stm32_usart_poll_put_char(struct uart_port *port, unsigned char ch)
1368 {
1369         stm32_usart_console_putchar(port, ch);
1370 }
1371 #endif /* CONFIG_CONSOLE_POLL */
1372
1373 static const struct uart_ops stm32_uart_ops = {
1374         .tx_empty       = stm32_usart_tx_empty,
1375         .set_mctrl      = stm32_usart_set_mctrl,
1376         .get_mctrl      = stm32_usart_get_mctrl,
1377         .stop_tx        = stm32_usart_stop_tx,
1378         .start_tx       = stm32_usart_start_tx,
1379         .throttle       = stm32_usart_throttle,
1380         .unthrottle     = stm32_usart_unthrottle,
1381         .stop_rx        = stm32_usart_stop_rx,
1382         .enable_ms      = stm32_usart_enable_ms,
1383         .break_ctl      = stm32_usart_break_ctl,
1384         .startup        = stm32_usart_startup,
1385         .shutdown       = stm32_usart_shutdown,
1386         .flush_buffer   = stm32_usart_flush_buffer,
1387         .set_termios    = stm32_usart_set_termios,
1388         .pm             = stm32_usart_pm,
1389         .type           = stm32_usart_type,
1390         .release_port   = stm32_usart_release_port,
1391         .request_port   = stm32_usart_request_port,
1392         .config_port    = stm32_usart_config_port,
1393         .verify_port    = stm32_usart_verify_port,
1394 #if defined(CONFIG_CONSOLE_POLL)
1395         .poll_init      = stm32_usart_poll_init,
1396         .poll_get_char  = stm32_usart_poll_get_char,
1397         .poll_put_char  = stm32_usart_poll_put_char,
1398 #endif /* CONFIG_CONSOLE_POLL */
1399 };
1400
1401 /*
1402  * STM32H7 RX & TX FIFO threshold configuration (CR3 RXFTCFG / TXFTCFG)
1403  * Note: 1 isn't a valid value in RXFTCFG / TXFTCFG. In this case,
1404  * RXNEIE / TXEIE can be used instead of threshold irqs: RXFTIE / TXFTIE.
1405  * So, RXFTCFG / TXFTCFG bitfields values are encoded as array index + 1.
1406  */
1407 static const u32 stm32h7_usart_fifo_thresh_cfg[] = { 1, 2, 4, 8, 12, 14, 16 };
1408
1409 static void stm32_usart_get_ftcfg(struct platform_device *pdev, const char *p,
1410                                   int *ftcfg)
1411 {
1412         u32 bytes, i;
1413
1414         /* DT option to get RX & TX FIFO threshold (default to 8 bytes) */
1415         if (of_property_read_u32(pdev->dev.of_node, p, &bytes))
1416                 bytes = 8;
1417
1418         for (i = 0; i < ARRAY_SIZE(stm32h7_usart_fifo_thresh_cfg); i++)
1419                 if (stm32h7_usart_fifo_thresh_cfg[i] >= bytes)
1420                         break;
1421         if (i >= ARRAY_SIZE(stm32h7_usart_fifo_thresh_cfg))
1422                 i = ARRAY_SIZE(stm32h7_usart_fifo_thresh_cfg) - 1;
1423
1424         dev_dbg(&pdev->dev, "%s set to %d bytes\n", p,
1425                 stm32h7_usart_fifo_thresh_cfg[i]);
1426
1427         /* Provide FIFO threshold ftcfg (1 is invalid: threshold irq unused) */
1428         if (i)
1429                 *ftcfg = i - 1;
1430         else
1431                 *ftcfg = -EINVAL;
1432 }
1433
1434 static void stm32_usart_deinit_port(struct stm32_port *stm32port)
1435 {
1436         clk_disable_unprepare(stm32port->clk);
1437 }
1438
1439 static const struct serial_rs485 stm32_rs485_supported = {
1440         .flags = SER_RS485_ENABLED | SER_RS485_RTS_ON_SEND | SER_RS485_RTS_AFTER_SEND |
1441                  SER_RS485_RX_DURING_TX,
1442         .delay_rts_before_send = 1,
1443         .delay_rts_after_send = 1,
1444 };
1445
1446 static int stm32_usart_init_port(struct stm32_port *stm32port,
1447                                  struct platform_device *pdev)
1448 {
1449         struct uart_port *port = &stm32port->port;
1450         struct resource *res;
1451         int ret, irq;
1452
1453         irq = platform_get_irq(pdev, 0);
1454         if (irq < 0)
1455                 return irq;
1456
1457         port->iotype    = UPIO_MEM;
1458         port->flags     = UPF_BOOT_AUTOCONF;
1459         port->ops       = &stm32_uart_ops;
1460         port->dev       = &pdev->dev;
1461         port->fifosize  = stm32port->info->cfg.fifosize;
1462         port->has_sysrq = IS_ENABLED(CONFIG_SERIAL_STM32_CONSOLE);
1463         port->irq = irq;
1464         port->rs485_config = stm32_usart_config_rs485;
1465         port->rs485_supported = stm32_rs485_supported;
1466
1467         ret = stm32_usart_init_rs485(port, pdev);
1468         if (ret)
1469                 return ret;
1470
1471         stm32port->wakeup_src = stm32port->info->cfg.has_wakeup &&
1472                 of_property_read_bool(pdev->dev.of_node, "wakeup-source");
1473
1474         stm32port->swap = stm32port->info->cfg.has_swap &&
1475                 of_property_read_bool(pdev->dev.of_node, "rx-tx-swap");
1476
1477         stm32port->fifoen = stm32port->info->cfg.has_fifo;
1478         if (stm32port->fifoen) {
1479                 stm32_usart_get_ftcfg(pdev, "rx-threshold",
1480                                       &stm32port->rxftcfg);
1481                 stm32_usart_get_ftcfg(pdev, "tx-threshold",
1482                                       &stm32port->txftcfg);
1483         }
1484
1485         port->membase = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
1486         if (IS_ERR(port->membase))
1487                 return PTR_ERR(port->membase);
1488         port->mapbase = res->start;
1489
1490         spin_lock_init(&port->lock);
1491
1492         stm32port->clk = devm_clk_get(&pdev->dev, NULL);
1493         if (IS_ERR(stm32port->clk))
1494                 return PTR_ERR(stm32port->clk);
1495
1496         /* Ensure that clk rate is correct by enabling the clk */
1497         ret = clk_prepare_enable(stm32port->clk);
1498         if (ret)
1499                 return ret;
1500
1501         stm32port->port.uartclk = clk_get_rate(stm32port->clk);
1502         if (!stm32port->port.uartclk) {
1503                 ret = -EINVAL;
1504                 goto err_clk;
1505         }
1506
1507         stm32port->gpios = mctrl_gpio_init(&stm32port->port, 0);
1508         if (IS_ERR(stm32port->gpios)) {
1509                 ret = PTR_ERR(stm32port->gpios);
1510                 goto err_clk;
1511         }
1512
1513         /*
1514          * Both CTS/RTS gpios and "st,hw-flow-ctrl" (deprecated) or "uart-has-rtscts"
1515          * properties should not be specified.
1516          */
1517         if (stm32port->hw_flow_control) {
1518                 if (mctrl_gpio_to_gpiod(stm32port->gpios, UART_GPIO_CTS) ||
1519                     mctrl_gpio_to_gpiod(stm32port->gpios, UART_GPIO_RTS)) {
1520                         dev_err(&pdev->dev, "Conflicting RTS/CTS config\n");
1521                         ret = -EINVAL;
1522                         goto err_clk;
1523                 }
1524         }
1525
1526         return ret;
1527
1528 err_clk:
1529         clk_disable_unprepare(stm32port->clk);
1530
1531         return ret;
1532 }
1533
1534 static struct stm32_port *stm32_usart_of_get_port(struct platform_device *pdev)
1535 {
1536         struct device_node *np = pdev->dev.of_node;
1537         int id;
1538
1539         if (!np)
1540                 return NULL;
1541
1542         id = of_alias_get_id(np, "serial");
1543         if (id < 0) {
1544                 dev_err(&pdev->dev, "failed to get alias id, errno %d\n", id);
1545                 return NULL;
1546         }
1547
1548         if (WARN_ON(id >= STM32_MAX_PORTS))
1549                 return NULL;
1550
1551         stm32_ports[id].hw_flow_control =
1552                 of_property_read_bool (np, "st,hw-flow-ctrl") /*deprecated*/ ||
1553                 of_property_read_bool (np, "uart-has-rtscts");
1554         stm32_ports[id].port.line = id;
1555         stm32_ports[id].cr1_irq = USART_CR1_RXNEIE;
1556         stm32_ports[id].cr3_irq = 0;
1557         stm32_ports[id].last_res = RX_BUF_L;
1558         return &stm32_ports[id];
1559 }
1560
1561 #ifdef CONFIG_OF
1562 static const struct of_device_id stm32_match[] = {
1563         { .compatible = "st,stm32-uart", .data = &stm32f4_info},
1564         { .compatible = "st,stm32f7-uart", .data = &stm32f7_info},
1565         { .compatible = "st,stm32h7-uart", .data = &stm32h7_info},
1566         {},
1567 };
1568
1569 MODULE_DEVICE_TABLE(of, stm32_match);
1570 #endif
1571
1572 static void stm32_usart_of_dma_rx_remove(struct stm32_port *stm32port,
1573                                          struct platform_device *pdev)
1574 {
1575         if (stm32port->rx_buf)
1576                 dma_free_coherent(&pdev->dev, RX_BUF_L, stm32port->rx_buf,
1577                                   stm32port->rx_dma_buf);
1578 }
1579
1580 static int stm32_usart_of_dma_rx_probe(struct stm32_port *stm32port,
1581                                        struct platform_device *pdev)
1582 {
1583         const struct stm32_usart_offsets *ofs = &stm32port->info->ofs;
1584         struct uart_port *port = &stm32port->port;
1585         struct device *dev = &pdev->dev;
1586         struct dma_slave_config config;
1587         int ret;
1588
1589         stm32port->rx_buf = dma_alloc_coherent(dev, RX_BUF_L,
1590                                                &stm32port->rx_dma_buf,
1591                                                GFP_KERNEL);
1592         if (!stm32port->rx_buf)
1593                 return -ENOMEM;
1594
1595         /* Configure DMA channel */
1596         memset(&config, 0, sizeof(config));
1597         config.src_addr = port->mapbase + ofs->rdr;
1598         config.src_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
1599
1600         ret = dmaengine_slave_config(stm32port->rx_ch, &config);
1601         if (ret < 0) {
1602                 dev_err(dev, "rx dma channel config failed\n");
1603                 stm32_usart_of_dma_rx_remove(stm32port, pdev);
1604                 return ret;
1605         }
1606
1607         return 0;
1608 }
1609
1610 static void stm32_usart_of_dma_tx_remove(struct stm32_port *stm32port,
1611                                          struct platform_device *pdev)
1612 {
1613         if (stm32port->tx_buf)
1614                 dma_free_coherent(&pdev->dev, TX_BUF_L, stm32port->tx_buf,
1615                                   stm32port->tx_dma_buf);
1616 }
1617
1618 static int stm32_usart_of_dma_tx_probe(struct stm32_port *stm32port,
1619                                        struct platform_device *pdev)
1620 {
1621         const struct stm32_usart_offsets *ofs = &stm32port->info->ofs;
1622         struct uart_port *port = &stm32port->port;
1623         struct device *dev = &pdev->dev;
1624         struct dma_slave_config config;
1625         int ret;
1626
1627         stm32port->tx_buf = dma_alloc_coherent(dev, TX_BUF_L,
1628                                                &stm32port->tx_dma_buf,
1629                                                GFP_KERNEL);
1630         if (!stm32port->tx_buf)
1631                 return -ENOMEM;
1632
1633         /* Configure DMA channel */
1634         memset(&config, 0, sizeof(config));
1635         config.dst_addr = port->mapbase + ofs->tdr;
1636         config.dst_addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
1637
1638         ret = dmaengine_slave_config(stm32port->tx_ch, &config);
1639         if (ret < 0) {
1640                 dev_err(dev, "tx dma channel config failed\n");
1641                 stm32_usart_of_dma_tx_remove(stm32port, pdev);
1642                 return ret;
1643         }
1644
1645         return 0;
1646 }
1647
1648 static int stm32_usart_serial_probe(struct platform_device *pdev)
1649 {
1650         struct stm32_port *stm32port;
1651         int ret;
1652
1653         stm32port = stm32_usart_of_get_port(pdev);
1654         if (!stm32port)
1655                 return -ENODEV;
1656
1657         stm32port->info = of_device_get_match_data(&pdev->dev);
1658         if (!stm32port->info)
1659                 return -EINVAL;
1660
1661         stm32port->rx_ch = dma_request_chan(&pdev->dev, "rx");
1662         if (PTR_ERR(stm32port->rx_ch) == -EPROBE_DEFER)
1663                 return -EPROBE_DEFER;
1664
1665         /* Fall back in interrupt mode for any non-deferral error */
1666         if (IS_ERR(stm32port->rx_ch))
1667                 stm32port->rx_ch = NULL;
1668
1669         stm32port->tx_ch = dma_request_chan(&pdev->dev, "tx");
1670         if (PTR_ERR(stm32port->tx_ch) == -EPROBE_DEFER) {
1671                 ret = -EPROBE_DEFER;
1672                 goto err_dma_rx;
1673         }
1674         /* Fall back in interrupt mode for any non-deferral error */
1675         if (IS_ERR(stm32port->tx_ch))
1676                 stm32port->tx_ch = NULL;
1677
1678         ret = stm32_usart_init_port(stm32port, pdev);
1679         if (ret)
1680                 goto err_dma_tx;
1681
1682         if (stm32port->wakeup_src) {
1683                 device_set_wakeup_capable(&pdev->dev, true);
1684                 ret = dev_pm_set_wake_irq(&pdev->dev, stm32port->port.irq);
1685                 if (ret)
1686                         goto err_deinit_port;
1687         }
1688
1689         if (stm32port->rx_ch && stm32_usart_of_dma_rx_probe(stm32port, pdev)) {
1690                 /* Fall back in interrupt mode */
1691                 dma_release_channel(stm32port->rx_ch);
1692                 stm32port->rx_ch = NULL;
1693         }
1694
1695         if (stm32port->tx_ch && stm32_usart_of_dma_tx_probe(stm32port, pdev)) {
1696                 /* Fall back in interrupt mode */
1697                 dma_release_channel(stm32port->tx_ch);
1698                 stm32port->tx_ch = NULL;
1699         }
1700
1701         if (!stm32port->rx_ch)
1702                 dev_info(&pdev->dev, "interrupt mode for rx (no dma)\n");
1703         if (!stm32port->tx_ch)
1704                 dev_info(&pdev->dev, "interrupt mode for tx (no dma)\n");
1705
1706         platform_set_drvdata(pdev, &stm32port->port);
1707
1708         pm_runtime_get_noresume(&pdev->dev);
1709         pm_runtime_set_active(&pdev->dev);
1710         pm_runtime_enable(&pdev->dev);
1711
1712         ret = uart_add_one_port(&stm32_usart_driver, &stm32port->port);
1713         if (ret)
1714                 goto err_port;
1715
1716         pm_runtime_put_sync(&pdev->dev);
1717
1718         return 0;
1719
1720 err_port:
1721         pm_runtime_disable(&pdev->dev);
1722         pm_runtime_set_suspended(&pdev->dev);
1723         pm_runtime_put_noidle(&pdev->dev);
1724
1725         if (stm32port->tx_ch)
1726                 stm32_usart_of_dma_tx_remove(stm32port, pdev);
1727         if (stm32port->rx_ch)
1728                 stm32_usart_of_dma_rx_remove(stm32port, pdev);
1729
1730         if (stm32port->wakeup_src)
1731                 dev_pm_clear_wake_irq(&pdev->dev);
1732
1733 err_deinit_port:
1734         if (stm32port->wakeup_src)
1735                 device_set_wakeup_capable(&pdev->dev, false);
1736
1737         stm32_usart_deinit_port(stm32port);
1738
1739 err_dma_tx:
1740         if (stm32port->tx_ch)
1741                 dma_release_channel(stm32port->tx_ch);
1742
1743 err_dma_rx:
1744         if (stm32port->rx_ch)
1745                 dma_release_channel(stm32port->rx_ch);
1746
1747         return ret;
1748 }
1749
1750 static int stm32_usart_serial_remove(struct platform_device *pdev)
1751 {
1752         struct uart_port *port = platform_get_drvdata(pdev);
1753         struct stm32_port *stm32_port = to_stm32_port(port);
1754         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
1755         int err;
1756         u32 cr3;
1757
1758         pm_runtime_get_sync(&pdev->dev);
1759         err = uart_remove_one_port(&stm32_usart_driver, port);
1760         if (err)
1761                 return(err);
1762
1763         pm_runtime_disable(&pdev->dev);
1764         pm_runtime_set_suspended(&pdev->dev);
1765         pm_runtime_put_noidle(&pdev->dev);
1766
1767         stm32_usart_clr_bits(port, ofs->cr1, USART_CR1_PEIE);
1768         cr3 = readl_relaxed(port->membase + ofs->cr3);
1769         cr3 &= ~USART_CR3_EIE;
1770         cr3 &= ~USART_CR3_DMAR;
1771         cr3 &= ~USART_CR3_DDRE;
1772         writel_relaxed(cr3, port->membase + ofs->cr3);
1773
1774         if (stm32_port->tx_ch) {
1775                 stm32_usart_of_dma_tx_remove(stm32_port, pdev);
1776                 dma_release_channel(stm32_port->tx_ch);
1777         }
1778
1779         if (stm32_port->rx_ch) {
1780                 stm32_usart_of_dma_rx_remove(stm32_port, pdev);
1781                 dma_release_channel(stm32_port->rx_ch);
1782         }
1783
1784         stm32_usart_clr_bits(port, ofs->cr3, USART_CR3_DMAT);
1785
1786         if (stm32_port->wakeup_src) {
1787                 dev_pm_clear_wake_irq(&pdev->dev);
1788                 device_init_wakeup(&pdev->dev, false);
1789         }
1790
1791         stm32_usart_deinit_port(stm32_port);
1792
1793         return 0;
1794 }
1795
1796 static void __maybe_unused stm32_usart_console_putchar(struct uart_port *port, unsigned char ch)
1797 {
1798         struct stm32_port *stm32_port = to_stm32_port(port);
1799         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
1800         u32 isr;
1801         int ret;
1802
1803         ret = readl_relaxed_poll_timeout_atomic(port->membase + ofs->isr, isr,
1804                                                 (isr & USART_SR_TXE), 100,
1805                                                 STM32_USART_TIMEOUT_USEC);
1806         if (ret != 0) {
1807                 dev_err(port->dev, "Error while sending data in UART TX : %d\n", ret);
1808                 return;
1809         }
1810         writel_relaxed(ch, port->membase + ofs->tdr);
1811 }
1812
1813 #ifdef CONFIG_SERIAL_STM32_CONSOLE
1814 static void stm32_usart_console_write(struct console *co, const char *s,
1815                                       unsigned int cnt)
1816 {
1817         struct uart_port *port = &stm32_ports[co->index].port;
1818         struct stm32_port *stm32_port = to_stm32_port(port);
1819         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
1820         const struct stm32_usart_config *cfg = &stm32_port->info->cfg;
1821         unsigned long flags;
1822         u32 old_cr1, new_cr1;
1823         int locked = 1;
1824
1825         if (oops_in_progress)
1826                 locked = spin_trylock_irqsave(&port->lock, flags);
1827         else
1828                 spin_lock_irqsave(&port->lock, flags);
1829
1830         /* Save and disable interrupts, enable the transmitter */
1831         old_cr1 = readl_relaxed(port->membase + ofs->cr1);
1832         new_cr1 = old_cr1 & ~USART_CR1_IE_MASK;
1833         new_cr1 |=  USART_CR1_TE | BIT(cfg->uart_enable_bit);
1834         writel_relaxed(new_cr1, port->membase + ofs->cr1);
1835
1836         uart_console_write(port, s, cnt, stm32_usart_console_putchar);
1837
1838         /* Restore interrupt state */
1839         writel_relaxed(old_cr1, port->membase + ofs->cr1);
1840
1841         if (locked)
1842                 spin_unlock_irqrestore(&port->lock, flags);
1843 }
1844
1845 static int stm32_usart_console_setup(struct console *co, char *options)
1846 {
1847         struct stm32_port *stm32port;
1848         int baud = 9600;
1849         int bits = 8;
1850         int parity = 'n';
1851         int flow = 'n';
1852
1853         if (co->index >= STM32_MAX_PORTS)
1854                 return -ENODEV;
1855
1856         stm32port = &stm32_ports[co->index];
1857
1858         /*
1859          * This driver does not support early console initialization
1860          * (use ARM early printk support instead), so we only expect
1861          * this to be called during the uart port registration when the
1862          * driver gets probed and the port should be mapped at that point.
1863          */
1864         if (stm32port->port.mapbase == 0 || !stm32port->port.membase)
1865                 return -ENXIO;
1866
1867         if (options)
1868                 uart_parse_options(options, &baud, &parity, &bits, &flow);
1869
1870         return uart_set_options(&stm32port->port, co, baud, parity, bits, flow);
1871 }
1872
1873 static struct console stm32_console = {
1874         .name           = STM32_SERIAL_NAME,
1875         .device         = uart_console_device,
1876         .write          = stm32_usart_console_write,
1877         .setup          = stm32_usart_console_setup,
1878         .flags          = CON_PRINTBUFFER,
1879         .index          = -1,
1880         .data           = &stm32_usart_driver,
1881 };
1882
1883 #define STM32_SERIAL_CONSOLE (&stm32_console)
1884
1885 #else
1886 #define STM32_SERIAL_CONSOLE NULL
1887 #endif /* CONFIG_SERIAL_STM32_CONSOLE */
1888
1889 #ifdef CONFIG_SERIAL_EARLYCON
1890 static void early_stm32_usart_console_putchar(struct uart_port *port, unsigned char ch)
1891 {
1892         struct stm32_usart_info *info = port->private_data;
1893
1894         while (!(readl_relaxed(port->membase + info->ofs.isr) & USART_SR_TXE))
1895                 cpu_relax();
1896
1897         writel_relaxed(ch, port->membase + info->ofs.tdr);
1898 }
1899
1900 static void early_stm32_serial_write(struct console *console, const char *s, unsigned int count)
1901 {
1902         struct earlycon_device *device = console->data;
1903         struct uart_port *port = &device->port;
1904
1905         uart_console_write(port, s, count, early_stm32_usart_console_putchar);
1906 }
1907
1908 static int __init early_stm32_h7_serial_setup(struct earlycon_device *device, const char *options)
1909 {
1910         if (!(device->port.membase || device->port.iobase))
1911                 return -ENODEV;
1912         device->port.private_data = &stm32h7_info;
1913         device->con->write = early_stm32_serial_write;
1914         return 0;
1915 }
1916
1917 static int __init early_stm32_f7_serial_setup(struct earlycon_device *device, const char *options)
1918 {
1919         if (!(device->port.membase || device->port.iobase))
1920                 return -ENODEV;
1921         device->port.private_data = &stm32f7_info;
1922         device->con->write = early_stm32_serial_write;
1923         return 0;
1924 }
1925
1926 static int __init early_stm32_f4_serial_setup(struct earlycon_device *device, const char *options)
1927 {
1928         if (!(device->port.membase || device->port.iobase))
1929                 return -ENODEV;
1930         device->port.private_data = &stm32f4_info;
1931         device->con->write = early_stm32_serial_write;
1932         return 0;
1933 }
1934
1935 OF_EARLYCON_DECLARE(stm32, "st,stm32h7-uart", early_stm32_h7_serial_setup);
1936 OF_EARLYCON_DECLARE(stm32, "st,stm32f7-uart", early_stm32_f7_serial_setup);
1937 OF_EARLYCON_DECLARE(stm32, "st,stm32-uart", early_stm32_f4_serial_setup);
1938 #endif /* CONFIG_SERIAL_EARLYCON */
1939
1940 static struct uart_driver stm32_usart_driver = {
1941         .driver_name    = DRIVER_NAME,
1942         .dev_name       = STM32_SERIAL_NAME,
1943         .major          = 0,
1944         .minor          = 0,
1945         .nr             = STM32_MAX_PORTS,
1946         .cons           = STM32_SERIAL_CONSOLE,
1947 };
1948
1949 static int __maybe_unused stm32_usart_serial_en_wakeup(struct uart_port *port,
1950                                                        bool enable)
1951 {
1952         struct stm32_port *stm32_port = to_stm32_port(port);
1953         const struct stm32_usart_offsets *ofs = &stm32_port->info->ofs;
1954         struct tty_port *tport = &port->state->port;
1955         int ret;
1956         unsigned int size;
1957         unsigned long flags;
1958
1959         if (!stm32_port->wakeup_src || !tty_port_initialized(tport))
1960                 return 0;
1961
1962         /*
1963          * Enable low-power wake-up and wake-up irq if argument is set to
1964          * "enable", disable low-power wake-up and wake-up irq otherwise
1965          */
1966         if (enable) {
1967                 stm32_usart_set_bits(port, ofs->cr1, USART_CR1_UESM);
1968                 stm32_usart_set_bits(port, ofs->cr3, USART_CR3_WUFIE);
1969                 mctrl_gpio_enable_irq_wake(stm32_port->gpios);
1970
1971                 /*
1972                  * When DMA is used for reception, it must be disabled before
1973                  * entering low-power mode and re-enabled when exiting from
1974                  * low-power mode.
1975                  */
1976                 if (stm32_port->rx_ch) {
1977                         spin_lock_irqsave(&port->lock, flags);
1978                         /* Avoid race with RX IRQ when DMAR is cleared */
1979                         stm32_usart_clr_bits(port, ofs->cr3, USART_CR3_DMAR);
1980                         /* Poll data from DMA RX buffer if any */
1981                         size = stm32_usart_receive_chars(port, true);
1982                         dmaengine_terminate_async(stm32_port->rx_ch);
1983                         uart_unlock_and_check_sysrq_irqrestore(port, flags);
1984                         if (size)
1985                                 tty_flip_buffer_push(tport);
1986                 }
1987
1988                 /* Poll data from RX FIFO if any */
1989                 stm32_usart_receive_chars(port, false);
1990         } else {
1991                 if (stm32_port->rx_ch) {
1992                         ret = stm32_usart_start_rx_dma_cyclic(port);
1993                         if (ret)
1994                                 return ret;
1995                 }
1996                 mctrl_gpio_disable_irq_wake(stm32_port->gpios);
1997                 stm32_usart_clr_bits(port, ofs->cr1, USART_CR1_UESM);
1998                 stm32_usart_clr_bits(port, ofs->cr3, USART_CR3_WUFIE);
1999         }
2000
2001         return 0;
2002 }
2003
2004 static int __maybe_unused stm32_usart_serial_suspend(struct device *dev)
2005 {
2006         struct uart_port *port = dev_get_drvdata(dev);
2007         int ret;
2008
2009         uart_suspend_port(&stm32_usart_driver, port);
2010
2011         if (device_may_wakeup(dev) || device_wakeup_path(dev)) {
2012                 ret = stm32_usart_serial_en_wakeup(port, true);
2013                 if (ret)
2014                         return ret;
2015         }
2016
2017         /*
2018          * When "no_console_suspend" is enabled, keep the pinctrl default state
2019          * and rely on bootloader stage to restore this state upon resume.
2020          * Otherwise, apply the idle or sleep states depending on wakeup
2021          * capabilities.
2022          */
2023         if (console_suspend_enabled || !uart_console(port)) {
2024                 if (device_may_wakeup(dev) || device_wakeup_path(dev))
2025                         pinctrl_pm_select_idle_state(dev);
2026                 else
2027                         pinctrl_pm_select_sleep_state(dev);
2028         }
2029
2030         return 0;
2031 }
2032
2033 static int __maybe_unused stm32_usart_serial_resume(struct device *dev)
2034 {
2035         struct uart_port *port = dev_get_drvdata(dev);
2036         int ret;
2037
2038         pinctrl_pm_select_default_state(dev);
2039
2040         if (device_may_wakeup(dev) || device_wakeup_path(dev)) {
2041                 ret = stm32_usart_serial_en_wakeup(port, false);
2042                 if (ret)
2043                         return ret;
2044         }
2045
2046         return uart_resume_port(&stm32_usart_driver, port);
2047 }
2048
2049 static int __maybe_unused stm32_usart_runtime_suspend(struct device *dev)
2050 {
2051         struct uart_port *port = dev_get_drvdata(dev);
2052         struct stm32_port *stm32port = container_of(port,
2053                         struct stm32_port, port);
2054
2055         clk_disable_unprepare(stm32port->clk);
2056
2057         return 0;
2058 }
2059
2060 static int __maybe_unused stm32_usart_runtime_resume(struct device *dev)
2061 {
2062         struct uart_port *port = dev_get_drvdata(dev);
2063         struct stm32_port *stm32port = container_of(port,
2064                         struct stm32_port, port);
2065
2066         return clk_prepare_enable(stm32port->clk);
2067 }
2068
2069 static const struct dev_pm_ops stm32_serial_pm_ops = {
2070         SET_RUNTIME_PM_OPS(stm32_usart_runtime_suspend,
2071                            stm32_usart_runtime_resume, NULL)
2072         SET_SYSTEM_SLEEP_PM_OPS(stm32_usart_serial_suspend,
2073                                 stm32_usart_serial_resume)
2074 };
2075
2076 static struct platform_driver stm32_serial_driver = {
2077         .probe          = stm32_usart_serial_probe,
2078         .remove         = stm32_usart_serial_remove,
2079         .driver = {
2080                 .name   = DRIVER_NAME,
2081                 .pm     = &stm32_serial_pm_ops,
2082                 .of_match_table = of_match_ptr(stm32_match),
2083         },
2084 };
2085
2086 static int __init stm32_usart_init(void)
2087 {
2088         static char banner[] __initdata = "STM32 USART driver initialized";
2089         int ret;
2090
2091         pr_info("%s\n", banner);
2092
2093         ret = uart_register_driver(&stm32_usart_driver);
2094         if (ret)
2095                 return ret;
2096
2097         ret = platform_driver_register(&stm32_serial_driver);
2098         if (ret)
2099                 uart_unregister_driver(&stm32_usart_driver);
2100
2101         return ret;
2102 }
2103
2104 static void __exit stm32_usart_exit(void)
2105 {
2106         platform_driver_unregister(&stm32_serial_driver);
2107         uart_unregister_driver(&stm32_usart_driver);
2108 }
2109
2110 module_init(stm32_usart_init);
2111 module_exit(stm32_usart_exit);
2112
2113 MODULE_ALIAS("platform:" DRIVER_NAME);
2114 MODULE_DESCRIPTION("STMicroelectronics STM32 serial port driver");
2115 MODULE_LICENSE("GPL v2");