drivers/tty/serial/qcom_geni_serial.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 // Copyright (c) 2017-2018, The Linux foundation. All rights reserved.
   3
   4 /* Disable MMIO tracing to prevent excessive logging of unwanted MMIO traces */
   5 #define __DISABLE_TRACE_MMIO__
   6
   7 #include <linux/clk.h>
   8 #include <linux/console.h>
   9 #include <linux/io.h>
  10 #include <linux/iopoll.h>
  11 #include <linux/irq.h>
  12 #include <linux/module.h>
  13 #include <linux/of.h>
  14 #include <linux/pm_opp.h>
  15 #include <linux/platform_device.h>
  16 #include <linux/pm_runtime.h>
  17 #include <linux/pm_wakeirq.h>
  18 #include <linux/soc/qcom/geni-se.h>
  19 #include <linux/serial.h>
  20 #include <linux/serial_core.h>
  21 #include <linux/slab.h>
  22 #include <linux/tty.h>
  23 #include <linux/tty_flip.h>
  24 #include <dt-bindings/interconnect/qcom,icc.h>
  25
  26 /* UART specific GENI registers */
  27 #define SE_UART_LOOPBACK_CFG            0x22c
  28 #define SE_UART_IO_MACRO_CTRL           0x240
  29 #define SE_UART_TX_TRANS_CFG            0x25c
  30 #define SE_UART_TX_WORD_LEN             0x268
  31 #define SE_UART_TX_STOP_BIT_LEN         0x26c
  32 #define SE_UART_TX_TRANS_LEN            0x270
  33 #define SE_UART_RX_TRANS_CFG            0x280
  34 #define SE_UART_RX_WORD_LEN             0x28c
  35 #define SE_UART_RX_STALE_CNT            0x294
  36 #define SE_UART_TX_PARITY_CFG           0x2a4
  37 #define SE_UART_RX_PARITY_CFG           0x2a8
  38 #define SE_UART_MANUAL_RFR              0x2ac
  39
  40 /* SE_UART_TRANS_CFG */
  41 #define UART_TX_PAR_EN                  BIT(0)
  42 #define UART_CTS_MASK                   BIT(1)
  43
  44 /* SE_UART_TX_STOP_BIT_LEN */
  45 #define TX_STOP_BIT_LEN_1               0
  46 #define TX_STOP_BIT_LEN_2               2
  47
  48 /* SE_UART_RX_TRANS_CFG */
  49 #define UART_RX_PAR_EN                  BIT(3)
  50
  51 /* SE_UART_RX_WORD_LEN */
  52 #define RX_WORD_LEN_MASK                GENMASK(9, 0)
  53
  54 /* SE_UART_RX_STALE_CNT */
  55 #define RX_STALE_CNT                    GENMASK(23, 0)
  56
  57 /* SE_UART_TX_PARITY_CFG/RX_PARITY_CFG */
  58 #define PAR_CALC_EN                     BIT(0)
  59 #define PAR_EVEN                        0x00
  60 #define PAR_ODD                         0x01
  61 #define PAR_SPACE                       0x10
  62
  63 /* SE_UART_MANUAL_RFR register fields */
  64 #define UART_MANUAL_RFR_EN              BIT(31)
  65 #define UART_RFR_NOT_READY              BIT(1)
  66 #define UART_RFR_READY                  BIT(0)
  67
  68 /* UART M_CMD OP codes */
  69 #define UART_START_TX                   0x1
  70 /* UART S_CMD OP codes */
  71 #define UART_START_READ                 0x1
  72 #define UART_PARAM                      0x1
  73 #define UART_PARAM_RFR_OPEN             BIT(7)
  74
  75 #define UART_OVERSAMPLING               32
  76 #define STALE_TIMEOUT                   16
  77 #define DEFAULT_BITS_PER_CHAR           10
  78 #define GENI_UART_CONS_PORTS            1
  79 #define GENI_UART_PORTS                 3
  80 #define DEF_FIFO_DEPTH_WORDS            16
  81 #define DEF_TX_WM                       2
  82 #define DEF_FIFO_WIDTH_BITS             32
  83 #define UART_RX_WM                      2
  84
  85 /* SE_UART_LOOPBACK_CFG */
  86 #define RX_TX_SORTED                    BIT(0)
  87 #define CTS_RTS_SORTED                  BIT(1)
  88 #define RX_TX_CTS_RTS_SORTED            (RX_TX_SORTED | CTS_RTS_SORTED)
  89
  90 /* UART pin swap value */
  91 #define DEFAULT_IO_MACRO_IO0_IO1_MASK   GENMASK(3, 0)
  92 #define IO_MACRO_IO0_SEL                0x3
  93 #define DEFAULT_IO_MACRO_IO2_IO3_MASK   GENMASK(15, 4)
  94 #define IO_MACRO_IO2_IO3_SWAP           0x4640
  95
  96 /* We always configure 4 bytes per FIFO word */
  97 #define BYTES_PER_FIFO_WORD             4U
  98
  99 #define DMA_RX_BUF_SIZE         2048
 100
 101 struct qcom_geni_device_data {
 102         bool console;
 103         enum geni_se_xfer_mode mode;
 104 };
 105
 106 struct qcom_geni_private_data {
 107         /* NOTE: earlycon port will have NULL here */
 108         struct uart_driver *drv;
 109
 110         u32 poll_cached_bytes;
 111         unsigned int poll_cached_bytes_cnt;
 112
 113         u32 write_cached_bytes;
 114         unsigned int write_cached_bytes_cnt;
 115 };
 116
 117 struct qcom_geni_serial_port {
 118         struct uart_port uport;
 119         struct geni_se se;
 120         const char *name;
 121         u32 tx_fifo_depth;
 122         u32 tx_fifo_width;
 123         u32 rx_fifo_depth;
 124         dma_addr_t tx_dma_addr;
 125         dma_addr_t rx_dma_addr;
 126         bool setup;
 127         unsigned long poll_timeout_us;
 128         unsigned long clk_rate;
 129         void *rx_buf;
 130         u32 loopback;
 131         bool brk;
 132
 133         unsigned int tx_remaining;
 134         unsigned int tx_queued;
 135         int wakeup_irq;
 136         bool rx_tx_swap;
 137         bool cts_rts_swap;
 138
 139         struct qcom_geni_private_data private_data;
 140         const struct qcom_geni_device_data *dev_data;
 141 };
 142
 143 static const struct uart_ops qcom_geni_console_pops;
 144 static const struct uart_ops qcom_geni_uart_pops;
 145 static struct uart_driver qcom_geni_console_driver;
 146 static struct uart_driver qcom_geni_uart_driver;
 147
 148 static void __qcom_geni_serial_cancel_tx_cmd(struct uart_port *uport);
 149 static void qcom_geni_serial_cancel_tx_cmd(struct uart_port *uport);
 150 static int qcom_geni_serial_port_setup(struct uart_port *uport);
 151
 152 static inline struct qcom_geni_serial_port *to_dev_port(struct uart_port *uport)
 153 {
 154         return container_of(uport, struct qcom_geni_serial_port, uport);
 155 }
 156
 157 static struct qcom_geni_serial_port qcom_geni_uart_ports[GENI_UART_PORTS] = {
 158         [0] = {
 159                 .uport = {
 160                         .iotype = UPIO_MEM,
 161                         .ops = &qcom_geni_uart_pops,
 162                         .flags = UPF_BOOT_AUTOCONF,
 163                         .line = 0,
 164                 },
 165         },
 166         [1] = {
 167                 .uport = {
 168                         .iotype = UPIO_MEM,
 169                         .ops = &qcom_geni_uart_pops,
 170                         .flags = UPF_BOOT_AUTOCONF,
 171                         .line = 1,
 172                 },
 173         },
 174         [2] = {
 175                 .uport = {
 176                         .iotype = UPIO_MEM,
 177                         .ops = &qcom_geni_uart_pops,
 178                         .flags = UPF_BOOT_AUTOCONF,
 179                         .line = 2,
 180                 },
 181         },
 182 };
 183
 184 static struct qcom_geni_serial_port qcom_geni_console_port = {
 185         .uport = {
 186                 .iotype = UPIO_MEM,
 187                 .ops = &qcom_geni_console_pops,
 188                 .flags = UPF_BOOT_AUTOCONF,
 189                 .line = 0,
 190         },
 191 };
 192
 193 static int qcom_geni_serial_request_port(struct uart_port *uport)
 194 {
 195         struct platform_device *pdev = to_platform_device(uport->dev);
 196         struct qcom_geni_serial_port *port = to_dev_port(uport);
 197
 198         uport->membase = devm_platform_ioremap_resource(pdev, 0);
 199         if (IS_ERR(uport->membase))
 200                 return PTR_ERR(uport->membase);
 201         port->se.base = uport->membase;
 202         return 0;
 203 }
 204
 205 static void qcom_geni_serial_config_port(struct uart_port *uport, int cfg_flags)
 206 {
 207         if (cfg_flags & UART_CONFIG_TYPE) {
 208                 uport->type = PORT_MSM;
 209                 qcom_geni_serial_request_port(uport);
 210         }
 211 }
 212
 213 static unsigned int qcom_geni_serial_get_mctrl(struct uart_port *uport)
 214 {
 215         unsigned int mctrl = TIOCM_DSR | TIOCM_CAR;
 216         u32 geni_ios;
 217
 218         if (uart_console(uport)) {
 219                 mctrl |= TIOCM_CTS;
 220         } else {
 221                 geni_ios = readl(uport->membase + SE_GENI_IOS);
 222                 if (!(geni_ios & IO2_DATA_IN))
 223                         mctrl |= TIOCM_CTS;
 224         }
 225
 226         return mctrl;
 227 }
 228
 229 static void qcom_geni_serial_set_mctrl(struct uart_port *uport,
 230                                                         unsigned int mctrl)
 231 {
 232         u32 uart_manual_rfr = 0;
 233         struct qcom_geni_serial_port *port = to_dev_port(uport);
 234
 235         if (uart_console(uport))
 236                 return;
 237
 238         if (mctrl & TIOCM_LOOP)
 239                 port->loopback = RX_TX_CTS_RTS_SORTED;
 240
 241         if (!(mctrl & TIOCM_RTS) && !uport->suspended)
 242                 uart_manual_rfr = UART_MANUAL_RFR_EN | UART_RFR_NOT_READY;
 243         writel(uart_manual_rfr, uport->membase + SE_UART_MANUAL_RFR);
 244 }
 245
 246 static const char *qcom_geni_serial_get_type(struct uart_port *uport)
 247 {
 248         return "MSM";
 249 }
 250
 251 static struct qcom_geni_serial_port *get_port_from_line(int line, bool console)
 252 {
 253         struct qcom_geni_serial_port *port;
 254         int nr_ports = console ? GENI_UART_CONS_PORTS : GENI_UART_PORTS;
 255
 256         if (line < 0 || line >= nr_ports)
 257                 return ERR_PTR(-ENXIO);
 258
 259         port = console ? &qcom_geni_console_port : &qcom_geni_uart_ports[line];
 260         return port;
 261 }
 262
 263 static bool qcom_geni_serial_main_active(struct uart_port *uport)
 264 {
 265         return readl(uport->membase + SE_GENI_STATUS) & M_GENI_CMD_ACTIVE;
 266 }
 267
 268 static bool qcom_geni_serial_secondary_active(struct uart_port *uport)
 269 {
 270         return readl(uport->membase + SE_GENI_STATUS) & S_GENI_CMD_ACTIVE;
 271 }
 272
 273 static bool qcom_geni_serial_poll_bitfield(struct uart_port *uport,
 274                                            unsigned int offset, u32 field, u32 val)
 275 {
 276         u32 reg;
 277         struct qcom_geni_serial_port *port;
 278         unsigned long timeout_us = 20000;
 279         struct qcom_geni_private_data *private_data = uport->private_data;
 280
 281         if (private_data->drv) {
 282                 port = to_dev_port(uport);
 283                 if (port->poll_timeout_us)
 284                         timeout_us = port->poll_timeout_us;
 285         }
 286
 287         /*
 288          * Use custom implementation instead of readl_poll_atomic since ktimer
 289          * is not ready at the time of early console.
 290          */
 291         timeout_us = DIV_ROUND_UP(timeout_us, 10) * 10;
 292         while (timeout_us) {
 293                 reg = readl(uport->membase + offset);
 294                 if ((reg & field) == val)
 295                         return true;
 296                 udelay(10);
 297                 timeout_us -= 10;
 298         }
 299         return false;
 300 }
 301
 302 static bool qcom_geni_serial_poll_bit(struct uart_port *uport,
 303                                       unsigned int offset, u32 field, bool set)
 304 {
 305         return qcom_geni_serial_poll_bitfield(uport, offset, field, set ? field : 0);
 306 }
 307
 308 static void qcom_geni_serial_setup_tx(struct uart_port *uport, u32 xmit_size)
 309 {
 310         u32 m_cmd;
 311
 312         writel(xmit_size, uport->membase + SE_UART_TX_TRANS_LEN);
 313         m_cmd = UART_START_TX << M_OPCODE_SHFT;
 314         writel(m_cmd, uport->membase + SE_GENI_M_CMD0);
 315 }
 316
 317 static void qcom_geni_serial_poll_tx_done(struct uart_port *uport)
 318 {
 319         int done;
 320
 321         done = qcom_geni_serial_poll_bit(uport, SE_GENI_M_IRQ_STATUS,
 322                                                 M_CMD_DONE_EN, true);
 323         if (!done) {
 324                 writel(M_GENI_CMD_ABORT, uport->membase +
 325                                                 SE_GENI_M_CMD_CTRL_REG);
 326                 qcom_geni_serial_poll_bit(uport, SE_GENI_M_IRQ_STATUS,
 327                                                         M_CMD_ABORT_EN, true);
 328                 writel(M_CMD_ABORT_EN, uport->membase + SE_GENI_M_IRQ_CLEAR);
 329         }
 330 }
 331
 332 static void qcom_geni_serial_abort_rx(struct uart_port *uport)
 333 {
 334         u32 irq_clear = S_CMD_DONE_EN | S_CMD_ABORT_EN;
 335
 336         writel(S_GENI_CMD_ABORT, uport->membase + SE_GENI_S_CMD_CTRL_REG);
 337         qcom_geni_serial_poll_bit(uport, SE_GENI_S_CMD_CTRL_REG,
 338                                         S_GENI_CMD_ABORT, false);
 339         writel(irq_clear, uport->membase + SE_GENI_S_IRQ_CLEAR);
 340         writel(FORCE_DEFAULT, uport->membase + GENI_FORCE_DEFAULT_REG);
 341 }
 342
 343 #ifdef CONFIG_CONSOLE_POLL
 344 static int qcom_geni_serial_get_char(struct uart_port *uport)
 345 {
 346         struct qcom_geni_private_data *private_data = uport->private_data;
 347         u32 status;
 348         u32 word_cnt;
 349         int ret;
 350
 351         if (!private_data->poll_cached_bytes_cnt) {
 352                 status = readl(uport->membase + SE_GENI_M_IRQ_STATUS);
 353                 writel(status, uport->membase + SE_GENI_M_IRQ_CLEAR);
 354
 355                 status = readl(uport->membase + SE_GENI_S_IRQ_STATUS);
 356                 writel(status, uport->membase + SE_GENI_S_IRQ_CLEAR);
 357
 358                 status = readl(uport->membase + SE_GENI_RX_FIFO_STATUS);
 359                 word_cnt = status & RX_FIFO_WC_MSK;
 360                 if (!word_cnt)
 361                         return NO_POLL_CHAR;
 362
 363                 if (word_cnt == 1 && (status & RX_LAST))
 364                         /*
 365                          * NOTE: If RX_LAST_BYTE_VALID is 0 it needs to be
 366                          * treated as if it was BYTES_PER_FIFO_WORD.
 367                          */
 368                         private_data->poll_cached_bytes_cnt =
 369                                 (status & RX_LAST_BYTE_VALID_MSK) >>
 370                                 RX_LAST_BYTE_VALID_SHFT;
 371
 372                 if (private_data->poll_cached_bytes_cnt == 0)
 373                         private_data->poll_cached_bytes_cnt = BYTES_PER_FIFO_WORD;
 374
 375                 private_data->poll_cached_bytes =
 376                         readl(uport->membase + SE_GENI_RX_FIFOn);
 377         }
 378
 379         private_data->poll_cached_bytes_cnt--;
 380         ret = private_data->poll_cached_bytes & 0xff;
 381         private_data->poll_cached_bytes >>= 8;
 382
 383         return ret;
 384 }
 385
 386 static void qcom_geni_serial_poll_put_char(struct uart_port *uport,
 387                                                         unsigned char c)
 388 {
 389         if (qcom_geni_serial_main_active(uport)) {
 390                 qcom_geni_serial_poll_tx_done(uport);
 391                 __qcom_geni_serial_cancel_tx_cmd(uport);
 392         }
 393
 394         writel(M_CMD_DONE_EN, uport->membase + SE_GENI_M_IRQ_CLEAR);
 395         qcom_geni_serial_setup_tx(uport, 1);
 396         writel(c, uport->membase + SE_GENI_TX_FIFOn);
 397         qcom_geni_serial_poll_tx_done(uport);
 398 }
 399
 400 static int qcom_geni_serial_poll_init(struct uart_port *uport)
 401 {
 402         struct qcom_geni_serial_port *port = to_dev_port(uport);
 403         int ret;
 404
 405         if (!port->setup) {
 406                 ret = qcom_geni_serial_port_setup(uport);
 407                 if (ret)
 408                         return ret;
 409         }
 410
 411         if (!qcom_geni_serial_secondary_active(uport))
 412                 geni_se_setup_s_cmd(&port->se, UART_START_READ, 0);
 413
 414         return 0;
 415 }
 416 #endif
 417
 418 #ifdef CONFIG_SERIAL_QCOM_GENI_CONSOLE
 419 static void qcom_geni_serial_drain_fifo(struct uart_port *uport)
 420 {
 421         struct qcom_geni_serial_port *port = to_dev_port(uport);
 422
 423         qcom_geni_serial_poll_bitfield(uport, SE_GENI_M_GP_LENGTH, GP_LENGTH,
 424                         port->tx_queued);
 425 }
 426
 427 static void qcom_geni_serial_wr_char(struct uart_port *uport, unsigned char ch)
 428 {
 429         struct qcom_geni_private_data *private_data = uport->private_data;
 430
 431         private_data->write_cached_bytes =
 432                 (private_data->write_cached_bytes >> 8) | (ch << 24);
 433         private_data->write_cached_bytes_cnt++;
 434
 435         if (private_data->write_cached_bytes_cnt == BYTES_PER_FIFO_WORD) {
 436                 writel(private_data->write_cached_bytes,
 437                        uport->membase + SE_GENI_TX_FIFOn);
 438                 private_data->write_cached_bytes_cnt = 0;
 439         }
 440 }
 441
 442 static void
 443 __qcom_geni_serial_console_write(struct uart_port *uport, const char *s,
 444                                  unsigned int count)
 445 {
 446         struct qcom_geni_private_data *private_data = uport->private_data;
 447
 448         int i;
 449         u32 bytes_to_send = count;
 450
 451         for (i = 0; i < count; i++) {
 452                 /*
 453                  * uart_console_write() adds a carriage return for each newline.
 454                  * Account for additional bytes to be written.
 455                  */
 456                 if (s[i] == '\n')
 457                         bytes_to_send++;
 458         }
 459
 460         writel(DEF_TX_WM, uport->membase + SE_GENI_TX_WATERMARK_REG);
 461         writel(M_CMD_DONE_EN, uport->membase + SE_GENI_M_IRQ_CLEAR);
 462         qcom_geni_serial_setup_tx(uport, bytes_to_send);
 463         for (i = 0; i < count; ) {
 464                 size_t chars_to_write = 0;
 465                 size_t avail = DEF_FIFO_DEPTH_WORDS - DEF_TX_WM;
 466
 467                 /*
 468                  * If the WM bit never set, then the Tx state machine is not
 469                  * in a valid state, so break, cancel/abort any existing
 470                  * command. Unfortunately the current data being written is
 471                  * lost.
 472                  */
 473                 if (!qcom_geni_serial_poll_bit(uport, SE_GENI_M_IRQ_STATUS,
 474                                                 M_TX_FIFO_WATERMARK_EN, true))
 475                         break;
 476                 chars_to_write = min_t(size_t, count - i, avail / 2);
 477                 uart_console_write(uport, s + i, chars_to_write,
 478                                                 qcom_geni_serial_wr_char);
 479                 writel(M_TX_FIFO_WATERMARK_EN, uport->membase +
 480                                                         SE_GENI_M_IRQ_CLEAR);
 481                 i += chars_to_write;
 482         }
 483
 484         if (private_data->write_cached_bytes_cnt) {
 485                 private_data->write_cached_bytes >>= BITS_PER_BYTE *
 486                         (BYTES_PER_FIFO_WORD - private_data->write_cached_bytes_cnt);
 487                 writel(private_data->write_cached_bytes,
 488                        uport->membase + SE_GENI_TX_FIFOn);
 489                 private_data->write_cached_bytes_cnt = 0;
 490         }
 491
 492         qcom_geni_serial_poll_tx_done(uport);
 493 }
 494
 495 static void qcom_geni_serial_console_write(struct console *co, const char *s,
 496                               unsigned int count)
 497 {
 498         struct uart_port *uport;
 499         struct qcom_geni_serial_port *port;
 500         u32 m_irq_en, s_irq_en;
 501         bool locked = true;
 502         unsigned long flags;
 503
 504         WARN_ON(co->index < 0 || co->index >= GENI_UART_CONS_PORTS);
 505
 506         port = get_port_from_line(co->index, true);
 507         if (IS_ERR(port))
 508                 return;
 509
 510         uport = &port->uport;
 511         if (oops_in_progress)
 512                 locked = uart_port_trylock_irqsave(uport, &flags);
 513         else
 514                 uart_port_lock_irqsave(uport, &flags);
 515
 516         m_irq_en = readl(uport->membase + SE_GENI_M_IRQ_EN);
 517         s_irq_en = readl(uport->membase + SE_GENI_S_IRQ_EN);
 518         writel(0, uport->membase + SE_GENI_M_IRQ_EN);
 519         writel(0, uport->membase + SE_GENI_S_IRQ_EN);
 520
 521         if (qcom_geni_serial_main_active(uport)) {
 522                 /* Wait for completion or drain FIFO */
 523                 if (!locked || port->tx_remaining == 0)
 524                         qcom_geni_serial_poll_tx_done(uport);
 525                 else
 526                         qcom_geni_serial_drain_fifo(uport);
 527
 528                 qcom_geni_serial_cancel_tx_cmd(uport);
 529         }
 530
 531         __qcom_geni_serial_console_write(uport, s, count);
 532
 533         writel(m_irq_en, uport->membase + SE_GENI_M_IRQ_EN);
 534         writel(s_irq_en, uport->membase + SE_GENI_S_IRQ_EN);
 535
 536         if (locked)
 537                 uart_port_unlock_irqrestore(uport, flags);
 538 }
 539
 540 static void handle_rx_console(struct uart_port *uport, u32 bytes, bool drop)
 541 {
 542         u32 i;
 543         unsigned char buf[sizeof(u32)];
 544         struct tty_port *tport;
 545         struct qcom_geni_serial_port *port = to_dev_port(uport);
 546
 547         tport = &uport->state->port;
 548         for (i = 0; i < bytes; ) {
 549                 int c;
 550                 int chunk = min_t(int, bytes - i, BYTES_PER_FIFO_WORD);
 551
 552                 ioread32_rep(uport->membase + SE_GENI_RX_FIFOn, buf, 1);
 553                 i += chunk;
 554                 if (drop)
 555                         continue;
 556
 557                 for (c = 0; c < chunk; c++) {
 558                         int sysrq;
 559
 560                         uport->icount.rx++;
 561                         if (port->brk && buf[c] == 0) {
 562                                 port->brk = false;
 563                                 if (uart_handle_break(uport))
 564                                         continue;
 565                         }
 566
 567                         sysrq = uart_prepare_sysrq_char(uport, buf[c]);
 568
 569                         if (!sysrq)
 570                                 tty_insert_flip_char(tport, buf[c], TTY_NORMAL);
 571                 }
 572         }
 573         if (!drop)
 574                 tty_flip_buffer_push(tport);
 575 }
 576 #else
 577 static void handle_rx_console(struct uart_port *uport, u32 bytes, bool drop)
 578 {
 579
 580 }
 581 #endif /* CONFIG_SERIAL_QCOM_GENI_CONSOLE */
 582
 583 static void handle_rx_uart(struct uart_port *uport, u32 bytes)
 584 {
 585         struct qcom_geni_serial_port *port = to_dev_port(uport);
 586         struct tty_port *tport = &uport->state->port;
 587         int ret;
 588
 589         ret = tty_insert_flip_string(tport, port->rx_buf, bytes);
 590         if (ret != bytes) {
 591                 dev_err_ratelimited(uport->dev, "failed to push data (%d < %u)\n",
 592                                 ret, bytes);
 593         }
 594         uport->icount.rx += ret;
 595         tty_flip_buffer_push(tport);
 596 }
 597
 598 static unsigned int qcom_geni_serial_tx_empty(struct uart_port *uport)
 599 {
 600         return !readl(uport->membase + SE_GENI_TX_FIFO_STATUS);
 601 }
 602
 603 static void qcom_geni_serial_stop_tx_dma(struct uart_port *uport)
 604 {
 605         struct qcom_geni_serial_port *port = to_dev_port(uport);
 606         bool done;
 607
 608         if (!qcom_geni_serial_main_active(uport))
 609                 return;
 610
 611         if (port->tx_dma_addr) {
 612                 geni_se_tx_dma_unprep(&port->se, port->tx_dma_addr,
 613                                       port->tx_remaining);
 614                 port->tx_dma_addr = 0;
 615                 port->tx_remaining = 0;
 616         }
 617
 618         geni_se_cancel_m_cmd(&port->se);
 619
 620         done = qcom_geni_serial_poll_bit(uport, SE_GENI_M_IRQ_STATUS,
 621                                          M_CMD_CANCEL_EN, true);
 622         if (!done) {
 623                 geni_se_abort_m_cmd(&port->se);
 624                 done = qcom_geni_serial_poll_bit(uport, SE_GENI_M_IRQ_STATUS,
 625                                                  M_CMD_ABORT_EN, true);
 626                 if (!done)
 627                         dev_err_ratelimited(uport->dev, "M_CMD_ABORT_EN not set");
 628                 writel(M_CMD_ABORT_EN, uport->membase + SE_GENI_M_IRQ_CLEAR);
 629         }
 630
 631         writel(M_CMD_CANCEL_EN, uport->membase + SE_GENI_M_IRQ_CLEAR);
 632 }
 633
 634 static void qcom_geni_serial_start_tx_dma(struct uart_port *uport)
 635 {
 636         struct qcom_geni_serial_port *port = to_dev_port(uport);
 637         struct tty_port *tport = &uport->state->port;
 638         unsigned int xmit_size;
 639         u8 *tail;
 640         int ret;
 641
 642         if (port->tx_dma_addr)
 643                 return;
 644
 645         if (kfifo_is_empty(&tport->xmit_fifo))
 646                 return;
 647
 648         xmit_size = kfifo_out_linear_ptr(&tport->xmit_fifo, &tail,
 649                         UART_XMIT_SIZE);
 650
 651         qcom_geni_serial_setup_tx(uport, xmit_size);
 652
 653         ret = geni_se_tx_dma_prep(&port->se, tail, xmit_size,
 654                                   &port->tx_dma_addr);
 655         if (ret) {
 656                 dev_err(uport->dev, "unable to start TX SE DMA: %d\n", ret);
 657                 qcom_geni_serial_stop_tx_dma(uport);
 658                 return;
 659         }
 660
 661         port->tx_remaining = xmit_size;
 662 }
 663
 664 static void qcom_geni_serial_start_tx_fifo(struct uart_port *uport)
 665 {
 666         unsigned char c;
 667         u32 irq_en;
 668
 669         /*
 670          * Start a new transfer in case the previous command was cancelled and
 671          * left data in the FIFO which may prevent the watermark interrupt
 672          * from triggering. Note that the stale data is discarded.
 673          */
 674         if (!qcom_geni_serial_main_active(uport) &&
 675             !qcom_geni_serial_tx_empty(uport)) {
 676                 if (uart_fifo_out(uport, &c, 1) == 1) {
 677                         writel(M_CMD_DONE_EN, uport->membase + SE_GENI_M_IRQ_CLEAR);
 678                         qcom_geni_serial_setup_tx(uport, 1);
 679                         writel(c, uport->membase + SE_GENI_TX_FIFOn);
 680                 }
 681         }
 682
 683         irq_en = readl(uport->membase + SE_GENI_M_IRQ_EN);
 684         irq_en |= M_TX_FIFO_WATERMARK_EN | M_CMD_DONE_EN;
 685         writel(DEF_TX_WM, uport->membase + SE_GENI_TX_WATERMARK_REG);
 686         writel(irq_en, uport->membase + SE_GENI_M_IRQ_EN);
 687 }
 688
 689 static void qcom_geni_serial_stop_tx_fifo(struct uart_port *uport)
 690 {
 691         u32 irq_en;
 692
 693         irq_en = readl(uport->membase + SE_GENI_M_IRQ_EN);
 694         irq_en &= ~(M_CMD_DONE_EN | M_TX_FIFO_WATERMARK_EN);
 695         writel(0, uport->membase + SE_GENI_TX_WATERMARK_REG);
 696         writel(irq_en, uport->membase + SE_GENI_M_IRQ_EN);
 697 }
 698
 699 static void __qcom_geni_serial_cancel_tx_cmd(struct uart_port *uport)
 700 {
 701         struct qcom_geni_serial_port *port = to_dev_port(uport);
 702
 703         geni_se_cancel_m_cmd(&port->se);
 704         if (!qcom_geni_serial_poll_bit(uport, SE_GENI_M_IRQ_STATUS,
 705                                                 M_CMD_CANCEL_EN, true)) {
 706                 geni_se_abort_m_cmd(&port->se);
 707                 qcom_geni_serial_poll_bit(uport, SE_GENI_M_IRQ_STATUS,
 708                                                 M_CMD_ABORT_EN, true);
 709                 writel(M_CMD_ABORT_EN, uport->membase + SE_GENI_M_IRQ_CLEAR);
 710         }
 711         writel(M_CMD_CANCEL_EN, uport->membase + SE_GENI_M_IRQ_CLEAR);
 712 }
 713
 714 static void qcom_geni_serial_cancel_tx_cmd(struct uart_port *uport)
 715 {
 716         struct qcom_geni_serial_port *port = to_dev_port(uport);
 717
 718         if (!qcom_geni_serial_main_active(uport))
 719                 return;
 720
 721         __qcom_geni_serial_cancel_tx_cmd(uport);
 722
 723         port->tx_remaining = 0;
 724         port->tx_queued = 0;
 725 }
 726
 727 static void qcom_geni_serial_handle_rx_fifo(struct uart_port *uport, bool drop)
 728 {
 729         u32 status;
 730         u32 word_cnt;
 731         u32 last_word_byte_cnt;
 732         u32 last_word_partial;
 733         u32 total_bytes;
 734
 735         status = readl(uport->membase + SE_GENI_RX_FIFO_STATUS);
 736         word_cnt = status & RX_FIFO_WC_MSK;
 737         last_word_partial = status & RX_LAST;
 738         last_word_byte_cnt = (status & RX_LAST_BYTE_VALID_MSK) >>
 739                                                 RX_LAST_BYTE_VALID_SHFT;
 740
 741         if (!word_cnt)
 742                 return;
 743         total_bytes = BYTES_PER_FIFO_WORD * (word_cnt - 1);
 744         if (last_word_partial && last_word_byte_cnt)
 745                 total_bytes += last_word_byte_cnt;
 746         else
 747                 total_bytes += BYTES_PER_FIFO_WORD;
 748         handle_rx_console(uport, total_bytes, drop);
 749 }
 750
 751 static void qcom_geni_serial_stop_rx_fifo(struct uart_port *uport)
 752 {
 753         u32 irq_en;
 754         struct qcom_geni_serial_port *port = to_dev_port(uport);
 755         u32 s_irq_status;
 756
 757         irq_en = readl(uport->membase + SE_GENI_S_IRQ_EN);
 758         irq_en &= ~(S_RX_FIFO_WATERMARK_EN | S_RX_FIFO_LAST_EN);
 759         writel(irq_en, uport->membase + SE_GENI_S_IRQ_EN);
 760
 761         irq_en = readl(uport->membase + SE_GENI_M_IRQ_EN);
 762         irq_en &= ~(M_RX_FIFO_WATERMARK_EN | M_RX_FIFO_LAST_EN);
 763         writel(irq_en, uport->membase + SE_GENI_M_IRQ_EN);
 764
 765         if (!qcom_geni_serial_secondary_active(uport))
 766                 return;
 767
 768         geni_se_cancel_s_cmd(&port->se);
 769         qcom_geni_serial_poll_bit(uport, SE_GENI_S_IRQ_STATUS,
 770                                         S_CMD_CANCEL_EN, true);
 771         /*
 772          * If timeout occurs secondary engine remains active
 773          * and Abort sequence is executed.
 774          */
 775         s_irq_status = readl(uport->membase + SE_GENI_S_IRQ_STATUS);
 776         /* Flush the Rx buffer */
 777         if (s_irq_status & S_RX_FIFO_LAST_EN)
 778                 qcom_geni_serial_handle_rx_fifo(uport, true);
 779         writel(s_irq_status, uport->membase + SE_GENI_S_IRQ_CLEAR);
 780
 781         if (qcom_geni_serial_secondary_active(uport))
 782                 qcom_geni_serial_abort_rx(uport);
 783 }
 784
 785 static void qcom_geni_serial_start_rx_fifo(struct uart_port *uport)
 786 {
 787         u32 irq_en;
 788         struct qcom_geni_serial_port *port = to_dev_port(uport);
 789
 790         if (qcom_geni_serial_secondary_active(uport))
 791                 qcom_geni_serial_stop_rx_fifo(uport);
 792
 793         geni_se_setup_s_cmd(&port->se, UART_START_READ, 0);
 794
 795         irq_en = readl(uport->membase + SE_GENI_S_IRQ_EN);
 796         irq_en |= S_RX_FIFO_WATERMARK_EN | S_RX_FIFO_LAST_EN;
 797         writel(irq_en, uport->membase + SE_GENI_S_IRQ_EN);
 798
 799         irq_en = readl(uport->membase + SE_GENI_M_IRQ_EN);
 800         irq_en |= M_RX_FIFO_WATERMARK_EN | M_RX_FIFO_LAST_EN;
 801         writel(irq_en, uport->membase + SE_GENI_M_IRQ_EN);
 802 }
 803
 804 static void qcom_geni_serial_stop_rx_dma(struct uart_port *uport)
 805 {
 806         struct qcom_geni_serial_port *port = to_dev_port(uport);
 807         bool done;
 808
 809         if (!qcom_geni_serial_secondary_active(uport))
 810                 return;
 811
 812         geni_se_cancel_s_cmd(&port->se);
 813         done = qcom_geni_serial_poll_bit(uport, SE_DMA_RX_IRQ_STAT,
 814                         RX_EOT, true);
 815         if (done) {
 816                 writel(RX_EOT | RX_DMA_DONE,
 817                                 uport->membase + SE_DMA_RX_IRQ_CLR);
 818         } else {
 819                 qcom_geni_serial_abort_rx(uport);
 820
 821                 writel(1, uport->membase + SE_DMA_RX_FSM_RST);
 822                 qcom_geni_serial_poll_bit(uport, SE_DMA_RX_IRQ_STAT,
 823                                 RX_RESET_DONE, true);
 824                 writel(RX_RESET_DONE | RX_DMA_DONE,
 825                                 uport->membase + SE_DMA_RX_IRQ_CLR);
 826         }
 827
 828         if (port->rx_dma_addr) {
 829                 geni_se_rx_dma_unprep(&port->se, port->rx_dma_addr,
 830                                       DMA_RX_BUF_SIZE);
 831                 port->rx_dma_addr = 0;
 832         }
 833 }
 834
 835 static void qcom_geni_serial_start_rx_dma(struct uart_port *uport)
 836 {
 837         struct qcom_geni_serial_port *port = to_dev_port(uport);
 838         int ret;
 839
 840         if (qcom_geni_serial_secondary_active(uport))
 841                 qcom_geni_serial_stop_rx_dma(uport);
 842
 843         geni_se_setup_s_cmd(&port->se, UART_START_READ, UART_PARAM_RFR_OPEN);
 844
 845         ret = geni_se_rx_dma_prep(&port->se, port->rx_buf,
 846                                   DMA_RX_BUF_SIZE,
 847                                   &port->rx_dma_addr);
 848         if (ret) {
 849                 dev_err(uport->dev, "unable to start RX SE DMA: %d\n", ret);
 850                 qcom_geni_serial_stop_rx_dma(uport);
 851         }
 852 }
 853
 854 static void qcom_geni_serial_handle_rx_dma(struct uart_port *uport, bool drop)
 855 {
 856         struct qcom_geni_serial_port *port = to_dev_port(uport);
 857         u32 rx_in;
 858         int ret;
 859
 860         if (!qcom_geni_serial_secondary_active(uport))
 861                 return;
 862
 863         if (!port->rx_dma_addr)
 864                 return;
 865
 866         geni_se_rx_dma_unprep(&port->se, port->rx_dma_addr, DMA_RX_BUF_SIZE);
 867         port->rx_dma_addr = 0;
 868
 869         rx_in = readl(uport->membase + SE_DMA_RX_LEN_IN);
 870         if (!rx_in) {
 871                 dev_warn(uport->dev, "serial engine reports 0 RX bytes in!\n");
 872                 return;
 873         }
 874
 875         if (!drop)
 876                 handle_rx_uart(uport, rx_in);
 877
 878         ret = geni_se_rx_dma_prep(&port->se, port->rx_buf,
 879                                   DMA_RX_BUF_SIZE,
 880                                   &port->rx_dma_addr);
 881         if (ret) {
 882                 dev_err(uport->dev, "unable to start RX SE DMA: %d\n", ret);
 883                 qcom_geni_serial_stop_rx_dma(uport);
 884         }
 885 }
 886
 887 static void qcom_geni_serial_start_rx(struct uart_port *uport)
 888 {
 889         uport->ops->start_rx(uport);
 890 }
 891
 892 static void qcom_geni_serial_stop_rx(struct uart_port *uport)
 893 {
 894         uport->ops->stop_rx(uport);
 895 }
 896
 897 static void qcom_geni_serial_stop_tx(struct uart_port *uport)
 898 {
 899         uport->ops->stop_tx(uport);
 900 }
 901
 902 static void qcom_geni_serial_send_chunk_fifo(struct uart_port *uport,
 903                                              unsigned int chunk)
 904 {
 905         struct qcom_geni_serial_port *port = to_dev_port(uport);
 906         unsigned int tx_bytes, remaining = chunk;
 907         u8 buf[BYTES_PER_FIFO_WORD];
 908
 909         while (remaining) {
 910                 memset(buf, 0, sizeof(buf));
 911                 tx_bytes = min(remaining, BYTES_PER_FIFO_WORD);
 912
 913                 uart_fifo_out(uport, buf, tx_bytes);
 914
 915                 iowrite32_rep(uport->membase + SE_GENI_TX_FIFOn, buf, 1);
 916
 917                 remaining -= tx_bytes;
 918                 port->tx_remaining -= tx_bytes;
 919         }
 920 }
 921
 922 static void qcom_geni_serial_handle_tx_fifo(struct uart_port *uport,
 923                                             bool done, bool active)
 924 {
 925         struct qcom_geni_serial_port *port = to_dev_port(uport);
 926         struct tty_port *tport = &uport->state->port;
 927         size_t avail;
 928         size_t pending;
 929         u32 status;
 930         u32 irq_en;
 931         unsigned int chunk;
 932
 933         status = readl(uport->membase + SE_GENI_TX_FIFO_STATUS);
 934
 935         /* Complete the current tx command before taking newly added data */
 936         if (active)
 937                 pending = port->tx_remaining;
 938         else
 939                 pending = kfifo_len(&tport->xmit_fifo);
 940
 941         /* All data has been transmitted or command has been cancelled */
 942         if (!pending && done) {
 943                 qcom_geni_serial_stop_tx_fifo(uport);
 944                 goto out_write_wakeup;
 945         }
 946
 947         if (active)
 948                 avail = port->tx_fifo_depth - (status & TX_FIFO_WC);
 949         else
 950                 avail = port->tx_fifo_depth;
 951
 952         avail *= BYTES_PER_FIFO_WORD;
 953
 954         chunk = min(avail, pending);
 955         if (!chunk)
 956                 goto out_write_wakeup;
 957
 958         if (!active) {
 959                 qcom_geni_serial_setup_tx(uport, pending);
 960                 port->tx_remaining = pending;
 961                 port->tx_queued = 0;
 962
 963                 irq_en = readl(uport->membase + SE_GENI_M_IRQ_EN);
 964                 if (!(irq_en & M_TX_FIFO_WATERMARK_EN))
 965                         writel(irq_en | M_TX_FIFO_WATERMARK_EN,
 966                                         uport->membase + SE_GENI_M_IRQ_EN);
 967         }
 968
 969         qcom_geni_serial_send_chunk_fifo(uport, chunk);
 970         port->tx_queued += chunk;
 971
 972         /*
 973          * The tx fifo watermark is level triggered and latched. Though we had
 974          * cleared it in qcom_geni_serial_isr it will have already reasserted
 975          * so we must clear it again here after our writes.
 976          */
 977         writel(M_TX_FIFO_WATERMARK_EN,
 978                         uport->membase + SE_GENI_M_IRQ_CLEAR);
 979
 980 out_write_wakeup:
 981         if (!port->tx_remaining) {
 982                 irq_en = readl(uport->membase + SE_GENI_M_IRQ_EN);
 983                 if (irq_en & M_TX_FIFO_WATERMARK_EN)
 984                         writel(irq_en & ~M_TX_FIFO_WATERMARK_EN,
 985                                         uport->membase + SE_GENI_M_IRQ_EN);
 986         }
 987
 988         if (kfifo_len(&tport->xmit_fifo) < WAKEUP_CHARS)
 989                 uart_write_wakeup(uport);
 990 }
 991
 992 static void qcom_geni_serial_handle_tx_dma(struct uart_port *uport)
 993 {
 994         struct qcom_geni_serial_port *port = to_dev_port(uport);
 995         struct tty_port *tport = &uport->state->port;
 996
 997         uart_xmit_advance(uport, port->tx_remaining);
 998         geni_se_tx_dma_unprep(&port->se, port->tx_dma_addr, port->tx_remaining);
 999         port->tx_dma_addr = 0;
1000         port->tx_remaining = 0;
1001
1002         if (!kfifo_is_empty(&tport->xmit_fifo))
1003                 qcom_geni_serial_start_tx_dma(uport);
1004
1005         if (kfifo_len(&tport->xmit_fifo) < WAKEUP_CHARS)
1006                 uart_write_wakeup(uport);
1007 }
1008
1009 static irqreturn_t qcom_geni_serial_isr(int isr, void *dev)
1010 {
1011         u32 m_irq_en;
1012         u32 m_irq_status;
1013         u32 s_irq_status;
1014         u32 geni_status;
1015         u32 dma;
1016         u32 dma_tx_status;
1017         u32 dma_rx_status;
1018         struct uart_port *uport = dev;
1019         bool drop_rx = false;
1020         struct tty_port *tport = &uport->state->port;
1021         struct qcom_geni_serial_port *port = to_dev_port(uport);
1022
1023         if (uport->suspended)
1024                 return IRQ_NONE;
1025
1026         uart_port_lock(uport);
1027
1028         m_irq_status = readl(uport->membase + SE_GENI_M_IRQ_STATUS);
1029         s_irq_status = readl(uport->membase + SE_GENI_S_IRQ_STATUS);
1030         dma_tx_status = readl(uport->membase + SE_DMA_TX_IRQ_STAT);
1031         dma_rx_status = readl(uport->membase + SE_DMA_RX_IRQ_STAT);
1032         geni_status = readl(uport->membase + SE_GENI_STATUS);
1033         dma = readl(uport->membase + SE_GENI_DMA_MODE_EN);
1034         m_irq_en = readl(uport->membase + SE_GENI_M_IRQ_EN);
1035         writel(m_irq_status, uport->membase + SE_GENI_M_IRQ_CLEAR);
1036         writel(s_irq_status, uport->membase + SE_GENI_S_IRQ_CLEAR);
1037         writel(dma_tx_status, uport->membase + SE_DMA_TX_IRQ_CLR);
1038         writel(dma_rx_status, uport->membase + SE_DMA_RX_IRQ_CLR);
1039
1040         if (WARN_ON(m_irq_status & M_ILLEGAL_CMD_EN))
1041                 goto out_unlock;
1042
1043         if (s_irq_status & S_RX_FIFO_WR_ERR_EN) {
1044                 uport->icount.overrun++;
1045                 tty_insert_flip_char(tport, 0, TTY_OVERRUN);
1046         }
1047
1048         if (s_irq_status & (S_GP_IRQ_0_EN | S_GP_IRQ_1_EN)) {
1049                 if (s_irq_status & S_GP_IRQ_0_EN)
1050                         uport->icount.parity++;
1051                 drop_rx = true;
1052         } else if (s_irq_status & (S_GP_IRQ_2_EN | S_GP_IRQ_3_EN)) {
1053                 uport->icount.brk++;
1054                 port->brk = true;
1055         }
1056
1057         if (dma) {
1058                 if (dma_tx_status & TX_DMA_DONE)
1059                         qcom_geni_serial_handle_tx_dma(uport);
1060
1061                 if (dma_rx_status) {
1062                         if (dma_rx_status & RX_RESET_DONE)
1063                                 goto out_unlock;
1064
1065                         if (dma_rx_status & RX_DMA_PARITY_ERR) {
1066                                 uport->icount.parity++;
1067                                 drop_rx = true;
1068                         }
1069
1070                         if (dma_rx_status & RX_DMA_BREAK)
1071                                 uport->icount.brk++;
1072
1073                         if (dma_rx_status & (RX_DMA_DONE | RX_EOT))
1074                                 qcom_geni_serial_handle_rx_dma(uport, drop_rx);
1075                 }
1076         } else {
1077                 if (m_irq_status & m_irq_en &
1078                     (M_TX_FIFO_WATERMARK_EN | M_CMD_DONE_EN))
1079                         qcom_geni_serial_handle_tx_fifo(uport,
1080                                         m_irq_status & M_CMD_DONE_EN,
1081                                         geni_status & M_GENI_CMD_ACTIVE);
1082
1083                 if (s_irq_status & (S_RX_FIFO_WATERMARK_EN | S_RX_FIFO_LAST_EN))
1084                         qcom_geni_serial_handle_rx_fifo(uport, drop_rx);
1085         }
1086
1087 out_unlock:
1088         uart_unlock_and_check_sysrq(uport);
1089
1090         return IRQ_HANDLED;
1091 }
1092
1093 static int setup_fifos(struct qcom_geni_serial_port *port)
1094 {
1095         struct uart_port *uport;
1096         u32 old_rx_fifo_depth = port->rx_fifo_depth;
1097
1098         uport = &port->uport;
1099         port->tx_fifo_depth = geni_se_get_tx_fifo_depth(&port->se);
1100         port->tx_fifo_width = geni_se_get_tx_fifo_width(&port->se);
1101         port->rx_fifo_depth = geni_se_get_rx_fifo_depth(&port->se);
1102         uport->fifosize =
1103                 (port->tx_fifo_depth * port->tx_fifo_width) / BITS_PER_BYTE;
1104
1105         if (port->rx_buf && (old_rx_fifo_depth != port->rx_fifo_depth) && port->rx_fifo_depth) {
1106                 /*
1107                  * Use krealloc rather than krealloc_array because rx_buf is
1108                  * accessed as 1 byte entries as well as 4 byte entries so it's
1109                  * not necessarily an array.
1110                  */
1111                 port->rx_buf = devm_krealloc(uport->dev, port->rx_buf,
1112                                              port->rx_fifo_depth * sizeof(u32),
1113                                              GFP_KERNEL);
1114                 if (!port->rx_buf)
1115                         return -ENOMEM;
1116         }
1117
1118         return 0;
1119 }
1120
1121
1122 static void qcom_geni_serial_shutdown(struct uart_port *uport)
1123 {
1124         disable_irq(uport->irq);
1125
1126         uart_port_lock_irq(uport);
1127         qcom_geni_serial_stop_tx(uport);
1128         qcom_geni_serial_stop_rx(uport);
1129
1130         qcom_geni_serial_cancel_tx_cmd(uport);
1131         uart_port_unlock_irq(uport);
1132 }
1133
1134 static void qcom_geni_serial_flush_buffer(struct uart_port *uport)
1135 {
1136         qcom_geni_serial_cancel_tx_cmd(uport);
1137 }
1138
1139 static int qcom_geni_serial_port_setup(struct uart_port *uport)
1140 {
1141         struct qcom_geni_serial_port *port = to_dev_port(uport);
1142         u32 rxstale = DEFAULT_BITS_PER_CHAR * STALE_TIMEOUT;
1143         u32 proto;
1144         u32 pin_swap;
1145         int ret;
1146
1147         proto = geni_se_read_proto(&port->se);
1148         if (proto != GENI_SE_UART) {
1149                 dev_err(uport->dev, "Invalid FW loaded, proto: %d\n", proto);
1150                 return -ENXIO;
1151         }
1152
1153         qcom_geni_serial_stop_rx(uport);
1154
1155         ret = setup_fifos(port);
1156         if (ret)
1157                 return ret;
1158
1159         writel(rxstale, uport->membase + SE_UART_RX_STALE_CNT);
1160
1161         pin_swap = readl(uport->membase + SE_UART_IO_MACRO_CTRL);
1162         if (port->rx_tx_swap) {
1163                 pin_swap &= ~DEFAULT_IO_MACRO_IO2_IO3_MASK;
1164                 pin_swap |= IO_MACRO_IO2_IO3_SWAP;
1165         }
1166         if (port->cts_rts_swap) {
1167                 pin_swap &= ~DEFAULT_IO_MACRO_IO0_IO1_MASK;
1168                 pin_swap |= IO_MACRO_IO0_SEL;
1169         }
1170         /* Configure this register if RX-TX, CTS-RTS pins are swapped */
1171         if (port->rx_tx_swap || port->cts_rts_swap)
1172                 writel(pin_swap, uport->membase + SE_UART_IO_MACRO_CTRL);
1173
1174         /*
1175          * Make an unconditional cancel on the main sequencer to reset
1176          * it else we could end up in data loss scenarios.
1177          */
1178         if (uart_console(uport))
1179                 qcom_geni_serial_poll_tx_done(uport);
1180         geni_se_config_packing(&port->se, BITS_PER_BYTE, BYTES_PER_FIFO_WORD,
1181                                false, true, true);
1182         geni_se_init(&port->se, UART_RX_WM, port->rx_fifo_depth - 2);
1183         geni_se_select_mode(&port->se, port->dev_data->mode);
1184         port->setup = true;
1185
1186         return 0;
1187 }
1188
1189 static int qcom_geni_serial_startup(struct uart_port *uport)
1190 {
1191         int ret;
1192         struct qcom_geni_serial_port *port = to_dev_port(uport);
1193
1194         if (!port->setup) {
1195                 ret = qcom_geni_serial_port_setup(uport);
1196                 if (ret)
1197                         return ret;
1198         }
1199
1200         uart_port_lock_irq(uport);
1201         qcom_geni_serial_start_rx(uport);
1202         uart_port_unlock_irq(uport);
1203
1204         enable_irq(uport->irq);
1205
1206         return 0;
1207 }
1208
1209 static unsigned long find_clk_rate_in_tol(struct clk *clk, unsigned int desired_clk,
1210                         unsigned int *clk_div, unsigned int percent_tol)
1211 {
1212         unsigned long freq;
1213         unsigned long div, maxdiv;
1214         u64 mult;
1215         unsigned long offset, abs_tol, achieved;
1216
1217         abs_tol = div_u64((u64)desired_clk * percent_tol, 100);
1218         maxdiv = CLK_DIV_MSK >> CLK_DIV_SHFT;
1219         div = 1;
1220         while (div <= maxdiv) {
1221                 mult = (u64)div * desired_clk;
1222                 if (mult != (unsigned long)mult)
1223                         break;
1224
1225                 offset = div * abs_tol;
1226                 freq = clk_round_rate(clk, mult - offset);
1227
1228                 /* Can only get lower if we're done */
1229                 if (freq < mult - offset)
1230                         break;
1231
1232                 /*
1233                  * Re-calculate div in case rounding skipped rates but we
1234                  * ended up at a good one, then check for a match.
1235                  */
1236                 div = DIV_ROUND_CLOSEST(freq, desired_clk);
1237                 achieved = DIV_ROUND_CLOSEST(freq, div);
1238                 if (achieved <= desired_clk + abs_tol &&
1239                     achieved >= desired_clk - abs_tol) {
1240                         *clk_div = div;
1241                         return freq;
1242                 }
1243
1244                 div = DIV_ROUND_UP(freq, desired_clk);
1245         }
1246
1247         return 0;
1248 }
1249
1250 static unsigned long get_clk_div_rate(struct clk *clk, unsigned int baud,
1251                         unsigned int sampling_rate, unsigned int *clk_div)
1252 {
1253         unsigned long ser_clk;
1254         unsigned long desired_clk;
1255
1256         desired_clk = baud * sampling_rate;
1257         if (!desired_clk)
1258                 return 0;
1259
1260         /*
1261          * try to find a clock rate within 2% tolerance, then within 5%
1262          */
1263         ser_clk = find_clk_rate_in_tol(clk, desired_clk, clk_div, 2);
1264         if (!ser_clk)
1265                 ser_clk = find_clk_rate_in_tol(clk, desired_clk, clk_div, 5);
1266
1267         return ser_clk;
1268 }
1269
1270 static void qcom_geni_serial_set_termios(struct uart_port *uport,
1271                                          struct ktermios *termios,
1272                                          const struct ktermios *old)
1273 {
1274         unsigned int baud;
1275         u32 bits_per_char;
1276         u32 tx_trans_cfg;
1277         u32 tx_parity_cfg;
1278         u32 rx_trans_cfg;
1279         u32 rx_parity_cfg;
1280         u32 stop_bit_len;
1281         unsigned int clk_div;
1282         u32 ser_clk_cfg;
1283         struct qcom_geni_serial_port *port = to_dev_port(uport);
1284         unsigned long clk_rate;
1285         u32 ver, sampling_rate;
1286         unsigned int avg_bw_core;
1287         unsigned long timeout;
1288
1289         /* baud rate */
1290         baud = uart_get_baud_rate(uport, termios, old, 300, 4000000);
1291
1292         sampling_rate = UART_OVERSAMPLING;
1293         /* Sampling rate is halved for IP versions >= 2.5 */
1294         ver = geni_se_get_qup_hw_version(&port->se);
1295         if (ver >= QUP_SE_VERSION_2_5)
1296                 sampling_rate /= 2;
1297
1298         clk_rate = get_clk_div_rate(port->se.clk, baud,
1299                 sampling_rate, &clk_div);
1300         if (!clk_rate) {
1301                 dev_err(port->se.dev,
1302                         "Couldn't find suitable clock rate for %u\n",
1303                         baud * sampling_rate);
1304                 return;
1305         }
1306
1307         dev_dbg(port->se.dev, "desired_rate = %u, clk_rate = %lu, clk_div = %u\n",
1308                         baud * sampling_rate, clk_rate, clk_div);
1309
1310         uport->uartclk = clk_rate;
1311         port->clk_rate = clk_rate;
1312         dev_pm_opp_set_rate(uport->dev, clk_rate);
1313         ser_clk_cfg = SER_CLK_EN;
1314         ser_clk_cfg |= clk_div << CLK_DIV_SHFT;
1315
1316         /*
1317          * Bump up BW vote on CPU and CORE path as driver supports FIFO mode
1318          * only.
1319          */
1320         avg_bw_core = (baud > 115200) ? Bps_to_icc(CORE_2X_50_MHZ)
1321                                                 : GENI_DEFAULT_BW;
1322         port->se.icc_paths[GENI_TO_CORE].avg_bw = avg_bw_core;
1323         port->se.icc_paths[CPU_TO_GENI].avg_bw = Bps_to_icc(baud);
1324         geni_icc_set_bw(&port->se);
1325
1326         /* parity */
1327         tx_trans_cfg = readl(uport->membase + SE_UART_TX_TRANS_CFG);
1328         tx_parity_cfg = readl(uport->membase + SE_UART_TX_PARITY_CFG);
1329         rx_trans_cfg = readl(uport->membase + SE_UART_RX_TRANS_CFG);
1330         rx_parity_cfg = readl(uport->membase + SE_UART_RX_PARITY_CFG);
1331         if (termios->c_cflag & PARENB) {
1332                 tx_trans_cfg |= UART_TX_PAR_EN;
1333                 rx_trans_cfg |= UART_RX_PAR_EN;
1334                 tx_parity_cfg |= PAR_CALC_EN;
1335                 rx_parity_cfg |= PAR_CALC_EN;
1336                 if (termios->c_cflag & PARODD) {
1337                         tx_parity_cfg |= PAR_ODD;
1338                         rx_parity_cfg |= PAR_ODD;
1339                 } else if (termios->c_cflag & CMSPAR) {
1340                         tx_parity_cfg |= PAR_SPACE;
1341                         rx_parity_cfg |= PAR_SPACE;
1342                 } else {
1343                         tx_parity_cfg |= PAR_EVEN;
1344                         rx_parity_cfg |= PAR_EVEN;
1345                 }
1346         } else {
1347                 tx_trans_cfg &= ~UART_TX_PAR_EN;
1348                 rx_trans_cfg &= ~UART_RX_PAR_EN;
1349                 tx_parity_cfg &= ~PAR_CALC_EN;
1350                 rx_parity_cfg &= ~PAR_CALC_EN;
1351         }
1352
1353         /* bits per char */
1354         bits_per_char = tty_get_char_size(termios->c_cflag);
1355
1356         /* stop bits */
1357         if (termios->c_cflag & CSTOPB)
1358                 stop_bit_len = TX_STOP_BIT_LEN_2;
1359         else
1360                 stop_bit_len = TX_STOP_BIT_LEN_1;
1361
1362         /* flow control, clear the CTS_MASK bit if using flow control. */
1363         if (termios->c_cflag & CRTSCTS)
1364                 tx_trans_cfg &= ~UART_CTS_MASK;
1365         else
1366                 tx_trans_cfg |= UART_CTS_MASK;
1367
1368         if (baud) {
1369                 uart_update_timeout(uport, termios->c_cflag, baud);
1370
1371                 /*
1372                  * Make sure that qcom_geni_serial_poll_bitfield() waits for
1373                  * the FIFO, two-word intermediate transfer register and shift
1374                  * register to clear.
1375                  *
1376                  * Note that uart_fifo_timeout() also adds a 20 ms margin.
1377                  */
1378                 timeout = jiffies_to_usecs(uart_fifo_timeout(uport));
1379                 timeout += 3 * timeout / port->tx_fifo_depth;
1380                 WRITE_ONCE(port->poll_timeout_us, timeout);
1381         }
1382
1383         if (!uart_console(uport))
1384                 writel(port->loopback,
1385                                 uport->membase + SE_UART_LOOPBACK_CFG);
1386         writel(tx_trans_cfg, uport->membase + SE_UART_TX_TRANS_CFG);
1387         writel(tx_parity_cfg, uport->membase + SE_UART_TX_PARITY_CFG);
1388         writel(rx_trans_cfg, uport->membase + SE_UART_RX_TRANS_CFG);
1389         writel(rx_parity_cfg, uport->membase + SE_UART_RX_PARITY_CFG);
1390         writel(bits_per_char, uport->membase + SE_UART_TX_WORD_LEN);
1391         writel(bits_per_char, uport->membase + SE_UART_RX_WORD_LEN);
1392         writel(stop_bit_len, uport->membase + SE_UART_TX_STOP_BIT_LEN);
1393         writel(ser_clk_cfg, uport->membase + GENI_SER_M_CLK_CFG);
1394         writel(ser_clk_cfg, uport->membase + GENI_SER_S_CLK_CFG);
1395 }
1396
1397 #ifdef CONFIG_SERIAL_QCOM_GENI_CONSOLE
1398 static int qcom_geni_console_setup(struct console *co, char *options)
1399 {
1400         struct uart_port *uport;
1401         struct qcom_geni_serial_port *port;
1402         int baud = 115200;
1403         int bits = 8;
1404         int parity = 'n';
1405         int flow = 'n';
1406         int ret;
1407
1408         if (co->index >= GENI_UART_CONS_PORTS  || co->index < 0)
1409                 return -ENXIO;
1410
1411         port = get_port_from_line(co->index, true);
1412         if (IS_ERR(port)) {
1413                 pr_err("Invalid line %d\n", co->index);
1414                 return PTR_ERR(port);
1415         }
1416
1417         uport = &port->uport;
1418
1419         if (unlikely(!uport->membase))
1420                 return -ENXIO;
1421
1422         if (!port->setup) {
1423                 ret = qcom_geni_serial_port_setup(uport);
1424                 if (ret)
1425                         return ret;
1426         }
1427
1428         if (options)
1429                 uart_parse_options(options, &baud, &parity, &bits, &flow);
1430
1431         return uart_set_options(uport, co, baud, parity, bits, flow);
1432 }
1433
1434 static void qcom_geni_serial_earlycon_write(struct console *con,
1435                                         const char *s, unsigned int n)
1436 {
1437         struct earlycon_device *dev = con->data;
1438
1439         __qcom_geni_serial_console_write(&dev->port, s, n);
1440 }
1441
1442 #ifdef CONFIG_CONSOLE_POLL
1443 static int qcom_geni_serial_earlycon_read(struct console *con,
1444                                           char *s, unsigned int n)
1445 {
1446         struct earlycon_device *dev = con->data;
1447         struct uart_port *uport = &dev->port;
1448         int num_read = 0;
1449         int ch;
1450
1451         while (num_read < n) {
1452                 ch = qcom_geni_serial_get_char(uport);
1453                 if (ch == NO_POLL_CHAR)
1454                         break;
1455                 s[num_read++] = ch;
1456         }
1457
1458         return num_read;
1459 }
1460
1461 static void __init qcom_geni_serial_enable_early_read(struct geni_se *se,
1462                                                       struct console *con)
1463 {
1464         geni_se_setup_s_cmd(se, UART_START_READ, 0);
1465         con->read = qcom_geni_serial_earlycon_read;
1466 }
1467 #else
1468 static inline void qcom_geni_serial_enable_early_read(struct geni_se *se,
1469                                                       struct console *con) { }
1470 #endif
1471
1472 static struct qcom_geni_private_data earlycon_private_data;
1473
1474 static int __init qcom_geni_serial_earlycon_setup(struct earlycon_device *dev,
1475                                                                 const char *opt)
1476 {
1477         struct uart_port *uport = &dev->port;
1478         u32 tx_trans_cfg;
1479         u32 tx_parity_cfg = 0;  /* Disable Tx Parity */
1480         u32 rx_trans_cfg = 0;
1481         u32 rx_parity_cfg = 0;  /* Disable Rx Parity */
1482         u32 stop_bit_len = 0;   /* Default stop bit length - 1 bit */
1483         u32 bits_per_char;
1484         struct geni_se se;
1485
1486         if (!uport->membase)
1487                 return -EINVAL;
1488
1489         uport->private_data = &earlycon_private_data;
1490
1491         memset(&se, 0, sizeof(se));
1492         se.base = uport->membase;
1493         if (geni_se_read_proto(&se) != GENI_SE_UART)
1494                 return -ENXIO;
1495         /*
1496          * Ignore Flow control.
1497          * n = 8.
1498          */
1499         tx_trans_cfg = UART_CTS_MASK;
1500         bits_per_char = BITS_PER_BYTE;
1501
1502         /*
1503          * Make an unconditional cancel on the main sequencer to reset
1504          * it else we could end up in data loss scenarios.
1505          */
1506         qcom_geni_serial_poll_tx_done(uport);
1507         qcom_geni_serial_abort_rx(uport);
1508         geni_se_config_packing(&se, BITS_PER_BYTE, BYTES_PER_FIFO_WORD,
1509                                false, true, true);
1510         geni_se_init(&se, DEF_FIFO_DEPTH_WORDS / 2, DEF_FIFO_DEPTH_WORDS - 2);
1511         geni_se_select_mode(&se, GENI_SE_FIFO);
1512
1513         writel(tx_trans_cfg, uport->membase + SE_UART_TX_TRANS_CFG);
1514         writel(tx_parity_cfg, uport->membase + SE_UART_TX_PARITY_CFG);
1515         writel(rx_trans_cfg, uport->membase + SE_UART_RX_TRANS_CFG);
1516         writel(rx_parity_cfg, uport->membase + SE_UART_RX_PARITY_CFG);
1517         writel(bits_per_char, uport->membase + SE_UART_TX_WORD_LEN);
1518         writel(bits_per_char, uport->membase + SE_UART_RX_WORD_LEN);
1519         writel(stop_bit_len, uport->membase + SE_UART_TX_STOP_BIT_LEN);
1520
1521         dev->con->write = qcom_geni_serial_earlycon_write;
1522         dev->con->setup = NULL;
1523         qcom_geni_serial_enable_early_read(&se, dev->con);
1524
1525         return 0;
1526 }
1527 OF_EARLYCON_DECLARE(qcom_geni, "qcom,geni-debug-uart",
1528                                 qcom_geni_serial_earlycon_setup);
1529
1530 static int __init console_register(struct uart_driver *drv)
1531 {
1532         return uart_register_driver(drv);
1533 }
1534
1535 static void console_unregister(struct uart_driver *drv)
1536 {
1537         uart_unregister_driver(drv);
1538 }
1539
1540 static struct console cons_ops = {
1541         .name = "ttyMSM",
1542         .write = qcom_geni_serial_console_write,
1543         .device = uart_console_device,
1544         .setup = qcom_geni_console_setup,
1545         .flags = CON_PRINTBUFFER,
1546         .index = -1,
1547         .data = &qcom_geni_console_driver,
1548 };
1549
1550 static struct uart_driver qcom_geni_console_driver = {
1551         .owner = THIS_MODULE,
1552         .driver_name = "qcom_geni_console",
1553         .dev_name = "ttyMSM",
1554         .nr =  GENI_UART_CONS_PORTS,
1555         .cons = &cons_ops,
1556 };
1557 #else
1558 static int console_register(struct uart_driver *drv)
1559 {
1560         return 0;
1561 }
1562
1563 static void console_unregister(struct uart_driver *drv)
1564 {
1565 }
1566 #endif /* CONFIG_SERIAL_QCOM_GENI_CONSOLE */
1567
1568 static struct uart_driver qcom_geni_uart_driver = {
1569         .owner = THIS_MODULE,
1570         .driver_name = "qcom_geni_uart",
1571         .dev_name = "ttyHS",
1572         .nr =  GENI_UART_PORTS,
1573 };
1574
1575 static void qcom_geni_serial_pm(struct uart_port *uport,
1576                 unsigned int new_state, unsigned int old_state)
1577 {
1578         struct qcom_geni_serial_port *port = to_dev_port(uport);
1579
1580         /* If we've never been called, treat it as off */
1581         if (old_state == UART_PM_STATE_UNDEFINED)
1582                 old_state = UART_PM_STATE_OFF;
1583
1584         if (new_state == UART_PM_STATE_ON && old_state == UART_PM_STATE_OFF) {
1585                 geni_icc_enable(&port->se);
1586                 if (port->clk_rate)
1587                         dev_pm_opp_set_rate(uport->dev, port->clk_rate);
1588                 geni_se_resources_on(&port->se);
1589         } else if (new_state == UART_PM_STATE_OFF &&
1590                         old_state == UART_PM_STATE_ON) {
1591                 geni_se_resources_off(&port->se);
1592                 dev_pm_opp_set_rate(uport->dev, 0);
1593                 geni_icc_disable(&port->se);
1594         }
1595 }
1596
1597 static const struct uart_ops qcom_geni_console_pops = {
1598         .tx_empty = qcom_geni_serial_tx_empty,
1599         .stop_tx = qcom_geni_serial_stop_tx_fifo,
1600         .start_tx = qcom_geni_serial_start_tx_fifo,
1601         .stop_rx = qcom_geni_serial_stop_rx_fifo,
1602         .start_rx = qcom_geni_serial_start_rx_fifo,
1603         .set_termios = qcom_geni_serial_set_termios,
1604         .startup = qcom_geni_serial_startup,
1605         .request_port = qcom_geni_serial_request_port,
1606         .config_port = qcom_geni_serial_config_port,
1607         .shutdown = qcom_geni_serial_shutdown,
1608         .flush_buffer = qcom_geni_serial_flush_buffer,
1609         .type = qcom_geni_serial_get_type,
1610         .set_mctrl = qcom_geni_serial_set_mctrl,
1611         .get_mctrl = qcom_geni_serial_get_mctrl,
1612 #ifdef CONFIG_CONSOLE_POLL
1613         .poll_get_char  = qcom_geni_serial_get_char,
1614         .poll_put_char  = qcom_geni_serial_poll_put_char,
1615         .poll_init = qcom_geni_serial_poll_init,
1616 #endif
1617         .pm = qcom_geni_serial_pm,
1618 };
1619
1620 static const struct uart_ops qcom_geni_uart_pops = {
1621         .tx_empty = qcom_geni_serial_tx_empty,
1622         .stop_tx = qcom_geni_serial_stop_tx_dma,
1623         .start_tx = qcom_geni_serial_start_tx_dma,
1624         .start_rx = qcom_geni_serial_start_rx_dma,
1625         .stop_rx = qcom_geni_serial_stop_rx_dma,
1626         .set_termios = qcom_geni_serial_set_termios,
1627         .startup = qcom_geni_serial_startup,
1628         .request_port = qcom_geni_serial_request_port,
1629         .config_port = qcom_geni_serial_config_port,
1630         .shutdown = qcom_geni_serial_shutdown,
1631         .type = qcom_geni_serial_get_type,
1632         .set_mctrl = qcom_geni_serial_set_mctrl,
1633         .get_mctrl = qcom_geni_serial_get_mctrl,
1634         .pm = qcom_geni_serial_pm,
1635 };
1636
1637 static int qcom_geni_serial_probe(struct platform_device *pdev)
1638 {
1639         int ret = 0;
1640         int line;
1641         struct qcom_geni_serial_port *port;
1642         struct uart_port *uport;
1643         struct resource *res;
1644         int irq;
1645         struct uart_driver *drv;
1646         const struct qcom_geni_device_data *data;
1647
1648         data = of_device_get_match_data(&pdev->dev);
1649         if (!data)
1650                 return -EINVAL;
1651
1652         if (data->console) {
1653                 drv = &qcom_geni_console_driver;
1654                 line = of_alias_get_id(pdev->dev.of_node, "serial");
1655         } else {
1656                 drv = &qcom_geni_uart_driver;
1657                 line = of_alias_get_id(pdev->dev.of_node, "serial");
1658                 if (line == -ENODEV) /* compat with non-standard aliases */
1659                         line = of_alias_get_id(pdev->dev.of_node, "hsuart");
1660         }
1661
1662         port = get_port_from_line(line, data->console);
1663         if (IS_ERR(port)) {
1664                 dev_err(&pdev->dev, "Invalid line %d\n", line);
1665                 return PTR_ERR(port);
1666         }
1667
1668         uport = &port->uport;
1669         /* Don't allow 2 drivers to access the same port */
1670         if (uport->private_data)
1671                 return -ENODEV;
1672
1673         uport->dev = &pdev->dev;
1674         port->dev_data = data;
1675         port->se.dev = &pdev->dev;
1676         port->se.wrapper = dev_get_drvdata(pdev->dev.parent);
1677         port->se.clk = devm_clk_get(&pdev->dev, "se");
1678         if (IS_ERR(port->se.clk)) {
1679                 ret = PTR_ERR(port->se.clk);
1680                 dev_err(&pdev->dev, "Err getting SE Core clk %d\n", ret);
1681                 return ret;
1682         }
1683
1684         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1685         if (!res)
1686                 return -EINVAL;
1687         uport->mapbase = res->start;
1688
1689         port->tx_fifo_depth = DEF_FIFO_DEPTH_WORDS;
1690         port->rx_fifo_depth = DEF_FIFO_DEPTH_WORDS;
1691         port->tx_fifo_width = DEF_FIFO_WIDTH_BITS;
1692
1693         if (!data->console) {
1694                 port->rx_buf = devm_kzalloc(uport->dev,
1695                                             DMA_RX_BUF_SIZE, GFP_KERNEL);
1696                 if (!port->rx_buf)
1697                         return -ENOMEM;
1698         }
1699
1700         ret = geni_icc_get(&port->se, NULL);
1701         if (ret)
1702                 return ret;
1703         port->se.icc_paths[GENI_TO_CORE].avg_bw = GENI_DEFAULT_BW;
1704         port->se.icc_paths[CPU_TO_GENI].avg_bw = GENI_DEFAULT_BW;
1705
1706         /* Set BW for register access */
1707         ret = geni_icc_set_bw(&port->se);
1708         if (ret)
1709                 return ret;
1710
1711         port->name = devm_kasprintf(uport->dev, GFP_KERNEL,
1712                         "qcom_geni_serial_%s%d",
1713                         uart_console(uport) ? "console" : "uart", uport->line);
1714         if (!port->name)
1715                 return -ENOMEM;
1716
1717         irq = platform_get_irq(pdev, 0);
1718         if (irq < 0)
1719                 return irq;
1720         uport->irq = irq;
1721         uport->has_sysrq = IS_ENABLED(CONFIG_SERIAL_QCOM_GENI_CONSOLE);
1722
1723         if (!data->console)
1724                 port->wakeup_irq = platform_get_irq_optional(pdev, 1);
1725
1726         if (of_property_read_bool(pdev->dev.of_node, "rx-tx-swap"))
1727                 port->rx_tx_swap = true;
1728
1729         if (of_property_read_bool(pdev->dev.of_node, "cts-rts-swap"))
1730                 port->cts_rts_swap = true;
1731
1732         ret = devm_pm_opp_set_clkname(&pdev->dev, "se");
1733         if (ret)
1734                 return ret;
1735         /* OPP table is optional */
1736         ret = devm_pm_opp_of_add_table(&pdev->dev);
1737         if (ret && ret != -ENODEV) {
1738                 dev_err(&pdev->dev, "invalid OPP table in device tree\n");
1739                 return ret;
1740         }
1741
1742         port->private_data.drv = drv;
1743         uport->private_data = &port->private_data;
1744         platform_set_drvdata(pdev, port);
1745
1746         irq_set_status_flags(uport->irq, IRQ_NOAUTOEN);
1747         ret = devm_request_irq(uport->dev, uport->irq, qcom_geni_serial_isr,
1748                         IRQF_TRIGGER_HIGH, port->name, uport);
1749         if (ret) {
1750                 dev_err(uport->dev, "Failed to get IRQ ret %d\n", ret);
1751                 return ret;
1752         }
1753
1754         ret = uart_add_one_port(drv, uport);
1755         if (ret)
1756                 return ret;
1757
1758         if (port->wakeup_irq > 0) {
1759                 device_init_wakeup(&pdev->dev, true);
1760                 ret = dev_pm_set_dedicated_wake_irq(&pdev->dev,
1761                                                 port->wakeup_irq);
1762                 if (ret) {
1763                         device_init_wakeup(&pdev->dev, false);
1764                         uart_remove_one_port(drv, uport);
1765                         return ret;
1766                 }
1767         }
1768
1769         return 0;
1770 }
1771
1772 static void qcom_geni_serial_remove(struct platform_device *pdev)
1773 {
1774         struct qcom_geni_serial_port *port = platform_get_drvdata(pdev);
1775         struct uart_driver *drv = port->private_data.drv;
1776
1777         dev_pm_clear_wake_irq(&pdev->dev);
1778         device_init_wakeup(&pdev->dev, false);
1779         uart_remove_one_port(drv, &port->uport);
1780 }
1781
1782 static int qcom_geni_serial_suspend(struct device *dev)
1783 {
1784         struct qcom_geni_serial_port *port = dev_get_drvdata(dev);
1785         struct uart_port *uport = &port->uport;
1786         struct qcom_geni_private_data *private_data = uport->private_data;
1787
1788         /*
1789          * This is done so we can hit the lowest possible state in suspend
1790          * even with no_console_suspend
1791          */
1792         if (uart_console(uport)) {
1793                 geni_icc_set_tag(&port->se, QCOM_ICC_TAG_ACTIVE_ONLY);
1794                 geni_icc_set_bw(&port->se);
1795         }
1796         return uart_suspend_port(private_data->drv, uport);
1797 }
1798
1799 static int qcom_geni_serial_resume(struct device *dev)
1800 {
1801         int ret;
1802         struct qcom_geni_serial_port *port = dev_get_drvdata(dev);
1803         struct uart_port *uport = &port->uport;
1804         struct qcom_geni_private_data *private_data = uport->private_data;
1805
1806         ret = uart_resume_port(private_data->drv, uport);
1807         if (uart_console(uport)) {
1808                 geni_icc_set_tag(&port->se, QCOM_ICC_TAG_ALWAYS);
1809                 geni_icc_set_bw(&port->se);
1810         }
1811         return ret;
1812 }
1813
1814 static const struct qcom_geni_device_data qcom_geni_console_data = {
1815         .console = true,
1816         .mode = GENI_SE_FIFO,
1817 };
1818
1819 static const struct qcom_geni_device_data qcom_geni_uart_data = {
1820         .console = false,
1821         .mode = GENI_SE_DMA,
1822 };
1823
1824 static const struct dev_pm_ops qcom_geni_serial_pm_ops = {
1825         SYSTEM_SLEEP_PM_OPS(qcom_geni_serial_suspend, qcom_geni_serial_resume)
1826 };
1827
1828 static const struct of_device_id qcom_geni_serial_match_table[] = {
1829         {
1830                 .compatible = "qcom,geni-debug-uart",
1831                 .data = &qcom_geni_console_data,
1832         },
1833         {
1834                 .compatible = "qcom,geni-uart",
1835                 .data = &qcom_geni_uart_data,
1836         },
1837         {}
1838 };
1839 MODULE_DEVICE_TABLE(of, qcom_geni_serial_match_table);
1840
1841 static struct platform_driver qcom_geni_serial_platform_driver = {
1842         .remove = qcom_geni_serial_remove,
1843         .probe = qcom_geni_serial_probe,
1844         .driver = {
1845                 .name = "qcom_geni_serial",
1846                 .of_match_table = qcom_geni_serial_match_table,
1847                 .pm = &qcom_geni_serial_pm_ops,
1848         },
1849 };
1850
1851 static int __init qcom_geni_serial_init(void)
1852 {
1853         int ret;
1854
1855         ret = console_register(&qcom_geni_console_driver);
1856         if (ret)
1857                 return ret;
1858
1859         ret = uart_register_driver(&qcom_geni_uart_driver);
1860         if (ret) {
1861                 console_unregister(&qcom_geni_console_driver);
1862                 return ret;
1863         }
1864
1865         ret = platform_driver_register(&qcom_geni_serial_platform_driver);
1866         if (ret) {
1867                 console_unregister(&qcom_geni_console_driver);
1868                 uart_unregister_driver(&qcom_geni_uart_driver);
1869         }
1870         return ret;
1871 }
1872 module_init(qcom_geni_serial_init);
1873
1874 static void __exit qcom_geni_serial_exit(void)
1875 {
1876         platform_driver_unregister(&qcom_geni_serial_platform_driver);
1877         console_unregister(&qcom_geni_console_driver);
1878         uart_unregister_driver(&qcom_geni_uart_driver);
1879 }
1880 module_exit(qcom_geni_serial_exit);
1881
1882 MODULE_DESCRIPTION("Serial driver for GENI based QUP cores");
1883 MODULE_LICENSE("GPL v2");