drivers/i3c/master/dw-i3c-master.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2018 Synopsys, Inc. and/or its affiliates.
   4  *
   5  * Author: Vitor Soares <[email protected]>
   6  */
   7
   8 #include <linux/bitops.h>
   9 #include <linux/clk.h>
  10 #include <linux/completion.h>
  11 #include <linux/err.h>
  12 #include <linux/errno.h>
  13 #include <linux/i3c/master.h>
  14 #include <linux/interrupt.h>
  15 #include <linux/ioport.h>
  16 #include <linux/iopoll.h>
  17 #include <linux/list.h>
  18 #include <linux/module.h>
  19 #include <linux/of.h>
  20 #include <linux/pinctrl/consumer.h>
  21 #include <linux/platform_device.h>
  22 #include <linux/pm_runtime.h>
  23 #include <linux/reset.h>
  24 #include <linux/slab.h>
  25
  26 #include "dw-i3c-master.h"
  27
  28 #define DEVICE_CTRL                     0x0
  29 #define DEV_CTRL_ENABLE                 BIT(31)
  30 #define DEV_CTRL_RESUME                 BIT(30)
  31 #define DEV_CTRL_HOT_JOIN_NACK          BIT(8)
  32 #define DEV_CTRL_I2C_SLAVE_PRESENT      BIT(7)
  33
  34 #define DEVICE_ADDR                     0x4
  35 #define DEV_ADDR_DYNAMIC_ADDR_VALID     BIT(31)
  36 #define DEV_ADDR_DYNAMIC(x)             (((x) << 16) & GENMASK(22, 16))
  37
  38 #define HW_CAPABILITY                   0x8
  39 #define COMMAND_QUEUE_PORT              0xc
  40 #define COMMAND_PORT_TOC                BIT(30)
  41 #define COMMAND_PORT_READ_TRANSFER      BIT(28)
  42 #define COMMAND_PORT_SDAP               BIT(27)
  43 #define COMMAND_PORT_ROC                BIT(26)
  44 #define COMMAND_PORT_SPEED(x)           (((x) << 21) & GENMASK(23, 21))
  45 #define COMMAND_PORT_DEV_INDEX(x)       (((x) << 16) & GENMASK(20, 16))
  46 #define COMMAND_PORT_CP                 BIT(15)
  47 #define COMMAND_PORT_CMD(x)             (((x) << 7) & GENMASK(14, 7))
  48 #define COMMAND_PORT_TID(x)             (((x) << 3) & GENMASK(6, 3))
  49
  50 #define COMMAND_PORT_ARG_DATA_LEN(x)    (((x) << 16) & GENMASK(31, 16))
  51 #define COMMAND_PORT_ARG_DATA_LEN_MAX   65536
  52 #define COMMAND_PORT_TRANSFER_ARG       0x01
  53
  54 #define COMMAND_PORT_SDA_DATA_BYTE_3(x) (((x) << 24) & GENMASK(31, 24))
  55 #define COMMAND_PORT_SDA_DATA_BYTE_2(x) (((x) << 16) & GENMASK(23, 16))
  56 #define COMMAND_PORT_SDA_DATA_BYTE_1(x) (((x) << 8) & GENMASK(15, 8))
  57 #define COMMAND_PORT_SDA_BYTE_STRB_3    BIT(5)
  58 #define COMMAND_PORT_SDA_BYTE_STRB_2    BIT(4)
  59 #define COMMAND_PORT_SDA_BYTE_STRB_1    BIT(3)
  60 #define COMMAND_PORT_SHORT_DATA_ARG     0x02
  61
  62 #define COMMAND_PORT_DEV_COUNT(x)       (((x) << 21) & GENMASK(25, 21))
  63 #define COMMAND_PORT_ADDR_ASSGN_CMD     0x03
  64
  65 #define RESPONSE_QUEUE_PORT             0x10
  66 #define RESPONSE_PORT_ERR_STATUS(x)     (((x) & GENMASK(31, 28)) >> 28)
  67 #define RESPONSE_NO_ERROR               0
  68 #define RESPONSE_ERROR_CRC              1
  69 #define RESPONSE_ERROR_PARITY           2
  70 #define RESPONSE_ERROR_FRAME            3
  71 #define RESPONSE_ERROR_IBA_NACK         4
  72 #define RESPONSE_ERROR_ADDRESS_NACK     5
  73 #define RESPONSE_ERROR_OVER_UNDER_FLOW  6
  74 #define RESPONSE_ERROR_TRANSF_ABORT     8
  75 #define RESPONSE_ERROR_I2C_W_NACK_ERR   9
  76 #define RESPONSE_PORT_TID(x)            (((x) & GENMASK(27, 24)) >> 24)
  77 #define RESPONSE_PORT_DATA_LEN(x)       ((x) & GENMASK(15, 0))
  78
  79 #define RX_TX_DATA_PORT                 0x14
  80 #define IBI_QUEUE_STATUS                0x18
  81 #define IBI_QUEUE_STATUS_IBI_ID(x)      (((x) & GENMASK(15, 8)) >> 8)
  82 #define IBI_QUEUE_STATUS_DATA_LEN(x)    ((x) & GENMASK(7, 0))
  83 #define IBI_QUEUE_IBI_ADDR(x)           (IBI_QUEUE_STATUS_IBI_ID(x) >> 1)
  84 #define IBI_QUEUE_IBI_RNW(x)            (IBI_QUEUE_STATUS_IBI_ID(x) & BIT(0))
  85 #define IBI_TYPE_MR(x)                                                         \
  86         ((IBI_QUEUE_IBI_ADDR(x) != I3C_HOT_JOIN_ADDR) && !IBI_QUEUE_IBI_RNW(x))
  87 #define IBI_TYPE_HJ(x)                                                         \
  88         ((IBI_QUEUE_IBI_ADDR(x) == I3C_HOT_JOIN_ADDR) && !IBI_QUEUE_IBI_RNW(x))
  89 #define IBI_TYPE_SIRQ(x)                                                        \
  90         ((IBI_QUEUE_IBI_ADDR(x) != I3C_HOT_JOIN_ADDR) && IBI_QUEUE_IBI_RNW(x))
  91
  92 #define QUEUE_THLD_CTRL                 0x1c
  93 #define QUEUE_THLD_CTRL_IBI_STAT_MASK   GENMASK(31, 24)
  94 #define QUEUE_THLD_CTRL_IBI_STAT(x)     (((x) - 1) << 24)
  95 #define QUEUE_THLD_CTRL_IBI_DATA_MASK   GENMASK(20, 16)
  96 #define QUEUE_THLD_CTRL_IBI_DATA(x)     ((x) << 16)
  97 #define QUEUE_THLD_CTRL_RESP_BUF_MASK   GENMASK(15, 8)
  98 #define QUEUE_THLD_CTRL_RESP_BUF(x)     (((x) - 1) << 8)
  99
 100 #define DATA_BUFFER_THLD_CTRL           0x20
 101 #define DATA_BUFFER_THLD_CTRL_RX_BUF    GENMASK(11, 8)
 102
 103 #define IBI_QUEUE_CTRL                  0x24
 104 #define IBI_MR_REQ_REJECT               0x2C
 105 #define IBI_SIR_REQ_REJECT              0x30
 106 #define IBI_REQ_REJECT_ALL              GENMASK(31, 0)
 107
 108 #define RESET_CTRL                      0x34
 109 #define RESET_CTRL_IBI_QUEUE            BIT(5)
 110 #define RESET_CTRL_RX_FIFO              BIT(4)
 111 #define RESET_CTRL_TX_FIFO              BIT(3)
 112 #define RESET_CTRL_RESP_QUEUE           BIT(2)
 113 #define RESET_CTRL_CMD_QUEUE            BIT(1)
 114 #define RESET_CTRL_SOFT                 BIT(0)
 115
 116 #define SLV_EVENT_CTRL                  0x38
 117 #define INTR_STATUS                     0x3c
 118 #define INTR_STATUS_EN                  0x40
 119 #define INTR_SIGNAL_EN                  0x44
 120 #define INTR_FORCE                      0x48
 121 #define INTR_BUSOWNER_UPDATE_STAT       BIT(13)
 122 #define INTR_IBI_UPDATED_STAT           BIT(12)
 123 #define INTR_READ_REQ_RECV_STAT         BIT(11)
 124 #define INTR_DEFSLV_STAT                BIT(10)
 125 #define INTR_TRANSFER_ERR_STAT          BIT(9)
 126 #define INTR_DYN_ADDR_ASSGN_STAT        BIT(8)
 127 #define INTR_CCC_UPDATED_STAT           BIT(6)
 128 #define INTR_TRANSFER_ABORT_STAT        BIT(5)
 129 #define INTR_RESP_READY_STAT            BIT(4)
 130 #define INTR_CMD_QUEUE_READY_STAT       BIT(3)
 131 #define INTR_IBI_THLD_STAT              BIT(2)
 132 #define INTR_RX_THLD_STAT               BIT(1)
 133 #define INTR_TX_THLD_STAT               BIT(0)
 134 #define INTR_ALL                        (INTR_BUSOWNER_UPDATE_STAT |    \
 135                                         INTR_IBI_UPDATED_STAT |         \
 136                                         INTR_READ_REQ_RECV_STAT |       \
 137                                         INTR_DEFSLV_STAT |              \
 138                                         INTR_TRANSFER_ERR_STAT |        \
 139                                         INTR_DYN_ADDR_ASSGN_STAT |      \
 140                                         INTR_CCC_UPDATED_STAT |         \
 141                                         INTR_TRANSFER_ABORT_STAT |      \
 142                                         INTR_RESP_READY_STAT |          \
 143                                         INTR_CMD_QUEUE_READY_STAT |     \
 144                                         INTR_IBI_THLD_STAT |            \
 145                                         INTR_TX_THLD_STAT |             \
 146                                         INTR_RX_THLD_STAT)
 147
 148 #define INTR_MASTER_MASK                (INTR_TRANSFER_ERR_STAT |       \
 149                                          INTR_RESP_READY_STAT)
 150
 151 #define QUEUE_STATUS_LEVEL              0x4c
 152 #define QUEUE_STATUS_IBI_STATUS_CNT(x)  (((x) & GENMASK(28, 24)) >> 24)
 153 #define QUEUE_STATUS_IBI_BUF_BLR(x)     (((x) & GENMASK(23, 16)) >> 16)
 154 #define QUEUE_STATUS_LEVEL_RESP(x)      (((x) & GENMASK(15, 8)) >> 8)
 155 #define QUEUE_STATUS_LEVEL_CMD(x)       ((x) & GENMASK(7, 0))
 156
 157 #define DATA_BUFFER_STATUS_LEVEL        0x50
 158 #define DATA_BUFFER_STATUS_LEVEL_TX(x)  ((x) & GENMASK(7, 0))
 159
 160 #define PRESENT_STATE                   0x54
 161 #define CCC_DEVICE_STATUS               0x58
 162 #define DEVICE_ADDR_TABLE_POINTER       0x5c
 163 #define DEVICE_ADDR_TABLE_DEPTH(x)      (((x) & GENMASK(31, 16)) >> 16)
 164 #define DEVICE_ADDR_TABLE_ADDR(x)       ((x) & GENMASK(7, 0))
 165
 166 #define DEV_CHAR_TABLE_POINTER          0x60
 167 #define VENDOR_SPECIFIC_REG_POINTER     0x6c
 168 #define SLV_PID_VALUE                   0x74
 169 #define SLV_CHAR_CTRL                   0x78
 170 #define SLV_MAX_LEN                     0x7c
 171 #define MAX_READ_TURNAROUND             0x80
 172 #define MAX_DATA_SPEED                  0x84
 173 #define SLV_DEBUG_STATUS                0x88
 174 #define SLV_INTR_REQ                    0x8c
 175 #define DEVICE_CTRL_EXTENDED            0xb0
 176 #define SCL_I3C_OD_TIMING               0xb4
 177 #define SCL_I3C_PP_TIMING               0xb8
 178 #define SCL_I3C_TIMING_HCNT(x)          (((x) << 16) & GENMASK(23, 16))
 179 #define SCL_I3C_TIMING_LCNT(x)          ((x) & GENMASK(7, 0))
 180 #define SCL_I3C_TIMING_CNT_MIN          5
 181
 182 #define SCL_I2C_FM_TIMING               0xbc
 183 #define SCL_I2C_FM_TIMING_HCNT(x)       (((x) << 16) & GENMASK(31, 16))
 184 #define SCL_I2C_FM_TIMING_LCNT(x)       ((x) & GENMASK(15, 0))
 185
 186 #define SCL_I2C_FMP_TIMING              0xc0
 187 #define SCL_I2C_FMP_TIMING_HCNT(x)      (((x) << 16) & GENMASK(23, 16))
 188 #define SCL_I2C_FMP_TIMING_LCNT(x)      ((x) & GENMASK(15, 0))
 189
 190 #define SCL_EXT_LCNT_TIMING             0xc8
 191 #define SCL_EXT_LCNT_4(x)               (((x) << 24) & GENMASK(31, 24))
 192 #define SCL_EXT_LCNT_3(x)               (((x) << 16) & GENMASK(23, 16))
 193 #define SCL_EXT_LCNT_2(x)               (((x) << 8) & GENMASK(15, 8))
 194 #define SCL_EXT_LCNT_1(x)               ((x) & GENMASK(7, 0))
 195
 196 #define SCL_EXT_TERMN_LCNT_TIMING       0xcc
 197 #define BUS_FREE_TIMING                 0xd4
 198 #define BUS_I3C_MST_FREE(x)             ((x) & GENMASK(15, 0))
 199
 200 #define BUS_IDLE_TIMING                 0xd8
 201 #define I3C_VER_ID                      0xe0
 202 #define I3C_VER_TYPE                    0xe4
 203 #define EXTENDED_CAPABILITY             0xe8
 204 #define SLAVE_CONFIG                    0xec
 205
 206 #define DEV_ADDR_TABLE_IBI_MDB          BIT(12)
 207 #define DEV_ADDR_TABLE_SIR_REJECT       BIT(13)
 208 #define DEV_ADDR_TABLE_LEGACY_I2C_DEV   BIT(31)
 209 #define DEV_ADDR_TABLE_DYNAMIC_ADDR(x)  (((x) << 16) & GENMASK(23, 16))
 210 #define DEV_ADDR_TABLE_STATIC_ADDR(x)   ((x) & GENMASK(6, 0))
 211 #define DEV_ADDR_TABLE_LOC(start, idx)  ((start) + ((idx) << 2))
 212
 213 #define I3C_BUS_SDR1_SCL_RATE           8000000
 214 #define I3C_BUS_SDR2_SCL_RATE           6000000
 215 #define I3C_BUS_SDR3_SCL_RATE           4000000
 216 #define I3C_BUS_SDR4_SCL_RATE           2000000
 217 #define I3C_BUS_I2C_FM_TLOW_MIN_NS      1300
 218 #define I3C_BUS_I2C_FMP_TLOW_MIN_NS     500
 219 #define I3C_BUS_THIGH_MAX_NS            41
 220
 221 #define XFER_TIMEOUT (msecs_to_jiffies(1000))
 222 #define RPM_AUTOSUSPEND_TIMEOUT 1000 /* ms */
 223
 224 /* Timing values to configure 12.5MHz frequency */
 225 #define AMD_I3C_OD_TIMING          0x4C007C
 226 #define AMD_I3C_PP_TIMING          0x8001A
 227
 228 /* List of quirks */
 229 #define AMD_I3C_OD_PP_TIMING            BIT(1)
 230
 231 struct dw_i3c_cmd {
 232         u32 cmd_lo;
 233         u32 cmd_hi;
 234         u16 tx_len;
 235         const void *tx_buf;
 236         u16 rx_len;
 237         void *rx_buf;
 238         u8 error;
 239 };
 240
 241 struct dw_i3c_xfer {
 242         struct list_head node;
 243         struct completion comp;
 244         int ret;
 245         unsigned int ncmds;
 246         struct dw_i3c_cmd cmds[] __counted_by(ncmds);
 247 };
 248
 249 struct dw_i3c_i2c_dev_data {
 250         u8 index;
 251         struct i3c_generic_ibi_pool *ibi_pool;
 252 };
 253
 254 static u8 even_parity(u8 p)
 255 {
 256         p ^= p >> 4;
 257         p &= 0xf;
 258
 259         return (0x9669 >> p) & 1;
 260 }
 261
 262 static bool dw_i3c_master_supports_ccc_cmd(struct i3c_master_controller *m,
 263                                            const struct i3c_ccc_cmd *cmd)
 264 {
 265         if (cmd->ndests > 1)
 266                 return false;
 267
 268         switch (cmd->id) {
 269         case I3C_CCC_ENEC(true):
 270         case I3C_CCC_ENEC(false):
 271         case I3C_CCC_DISEC(true):
 272         case I3C_CCC_DISEC(false):
 273         case I3C_CCC_ENTAS(0, true):
 274         case I3C_CCC_ENTAS(0, false):
 275         case I3C_CCC_RSTDAA(true):
 276         case I3C_CCC_RSTDAA(false):
 277         case I3C_CCC_ENTDAA:
 278         case I3C_CCC_SETMWL(true):
 279         case I3C_CCC_SETMWL(false):
 280         case I3C_CCC_SETMRL(true):
 281         case I3C_CCC_SETMRL(false):
 282         case I3C_CCC_ENTHDR(0):
 283         case I3C_CCC_SETDASA:
 284         case I3C_CCC_SETNEWDA:
 285         case I3C_CCC_GETMWL:
 286         case I3C_CCC_GETMRL:
 287         case I3C_CCC_GETPID:
 288         case I3C_CCC_GETBCR:
 289         case I3C_CCC_GETDCR:
 290         case I3C_CCC_GETSTATUS:
 291         case I3C_CCC_GETMXDS:
 292         case I3C_CCC_GETHDRCAP:
 293                 return true;
 294         default:
 295                 return false;
 296         }
 297 }
 298
 299 static inline struct dw_i3c_master *
 300 to_dw_i3c_master(struct i3c_master_controller *master)
 301 {
 302         return container_of(master, struct dw_i3c_master, base);
 303 }
 304
 305 static void dw_i3c_master_disable(struct dw_i3c_master *master)
 306 {
 307         writel(readl(master->regs + DEVICE_CTRL) & ~DEV_CTRL_ENABLE,
 308                master->regs + DEVICE_CTRL);
 309 }
 310
 311 static void dw_i3c_master_enable(struct dw_i3c_master *master)
 312 {
 313         u32 dev_ctrl;
 314
 315         dev_ctrl = readl(master->regs + DEVICE_CTRL);
 316         /* For now don't support Hot-Join */
 317         dev_ctrl |= DEV_CTRL_HOT_JOIN_NACK;
 318         if (master->i2c_slv_prsnt)
 319                 dev_ctrl |= DEV_CTRL_I2C_SLAVE_PRESENT;
 320         writel(dev_ctrl | DEV_CTRL_ENABLE,
 321                master->regs + DEVICE_CTRL);
 322 }
 323
 324 static int dw_i3c_master_get_addr_pos(struct dw_i3c_master *master, u8 addr)
 325 {
 326         int pos;
 327
 328         for (pos = 0; pos < master->maxdevs; pos++) {
 329                 if (addr == master->devs[pos].addr)
 330                         return pos;
 331         }
 332
 333         return -EINVAL;
 334 }
 335
 336 static int dw_i3c_master_get_free_pos(struct dw_i3c_master *master)
 337 {
 338         if (!(master->free_pos & GENMASK(master->maxdevs - 1, 0)))
 339                 return -ENOSPC;
 340
 341         return ffs(master->free_pos) - 1;
 342 }
 343
 344 static void dw_i3c_master_wr_tx_fifo(struct dw_i3c_master *master,
 345                                      const u8 *bytes, int nbytes)
 346 {
 347         writesl(master->regs + RX_TX_DATA_PORT, bytes, nbytes / 4);
 348         if (nbytes & 3) {
 349                 u32 tmp = 0;
 350
 351                 memcpy(&tmp, bytes + (nbytes & ~3), nbytes & 3);
 352                 writesl(master->regs + RX_TX_DATA_PORT, &tmp, 1);
 353         }
 354 }
 355
 356 static void dw_i3c_master_read_fifo(struct dw_i3c_master *master,
 357                                     int reg,  u8 *bytes, int nbytes)
 358 {
 359         readsl(master->regs + reg, bytes, nbytes / 4);
 360         if (nbytes & 3) {
 361                 u32 tmp;
 362
 363                 readsl(master->regs + reg, &tmp, 1);
 364                 memcpy(bytes + (nbytes & ~3), &tmp, nbytes & 3);
 365         }
 366 }
 367
 368 static void dw_i3c_master_read_rx_fifo(struct dw_i3c_master *master,
 369                                        u8 *bytes, int nbytes)
 370 {
 371         return dw_i3c_master_read_fifo(master, RX_TX_DATA_PORT, bytes, nbytes);
 372 }
 373
 374 static void dw_i3c_master_read_ibi_fifo(struct dw_i3c_master *master,
 375                                         u8 *bytes, int nbytes)
 376 {
 377         return dw_i3c_master_read_fifo(master, IBI_QUEUE_STATUS, bytes, nbytes);
 378 }
 379
 380 static struct dw_i3c_xfer *
 381 dw_i3c_master_alloc_xfer(struct dw_i3c_master *master, unsigned int ncmds)
 382 {
 383         struct dw_i3c_xfer *xfer;
 384
 385         xfer = kzalloc(struct_size(xfer, cmds, ncmds), GFP_KERNEL);
 386         if (!xfer)
 387                 return NULL;
 388
 389         INIT_LIST_HEAD(&xfer->node);
 390         xfer->ncmds = ncmds;
 391         xfer->ret = -ETIMEDOUT;
 392
 393         return xfer;
 394 }
 395
 396 static void dw_i3c_master_free_xfer(struct dw_i3c_xfer *xfer)
 397 {
 398         kfree(xfer);
 399 }
 400
 401 static void dw_i3c_master_start_xfer_locked(struct dw_i3c_master *master)
 402 {
 403         struct dw_i3c_xfer *xfer = master->xferqueue.cur;
 404         unsigned int i;
 405         u32 thld_ctrl;
 406
 407         if (!xfer)
 408                 return;
 409
 410         for (i = 0; i < xfer->ncmds; i++) {
 411                 struct dw_i3c_cmd *cmd = &xfer->cmds[i];
 412
 413                 dw_i3c_master_wr_tx_fifo(master, cmd->tx_buf, cmd->tx_len);
 414         }
 415
 416         thld_ctrl = readl(master->regs + QUEUE_THLD_CTRL);
 417         thld_ctrl &= ~QUEUE_THLD_CTRL_RESP_BUF_MASK;
 418         thld_ctrl |= QUEUE_THLD_CTRL_RESP_BUF(xfer->ncmds);
 419         writel(thld_ctrl, master->regs + QUEUE_THLD_CTRL);
 420
 421         for (i = 0; i < xfer->ncmds; i++) {
 422                 struct dw_i3c_cmd *cmd = &xfer->cmds[i];
 423
 424                 writel(cmd->cmd_hi, master->regs + COMMAND_QUEUE_PORT);
 425                 writel(cmd->cmd_lo, master->regs + COMMAND_QUEUE_PORT);
 426         }
 427 }
 428
 429 static void dw_i3c_master_enqueue_xfer(struct dw_i3c_master *master,
 430                                        struct dw_i3c_xfer *xfer)
 431 {
 432         unsigned long flags;
 433
 434         init_completion(&xfer->comp);
 435         spin_lock_irqsave(&master->xferqueue.lock, flags);
 436         if (master->xferqueue.cur) {
 437                 list_add_tail(&xfer->node, &master->xferqueue.list);
 438         } else {
 439                 master->xferqueue.cur = xfer;
 440                 dw_i3c_master_start_xfer_locked(master);
 441         }
 442         spin_unlock_irqrestore(&master->xferqueue.lock, flags);
 443 }
 444
 445 static void dw_i3c_master_dequeue_xfer_locked(struct dw_i3c_master *master,
 446                                               struct dw_i3c_xfer *xfer)
 447 {
 448         if (master->xferqueue.cur == xfer) {
 449                 u32 status;
 450
 451                 master->xferqueue.cur = NULL;
 452
 453                 writel(RESET_CTRL_RX_FIFO | RESET_CTRL_TX_FIFO |
 454                        RESET_CTRL_RESP_QUEUE | RESET_CTRL_CMD_QUEUE,
 455                        master->regs + RESET_CTRL);
 456
 457                 readl_poll_timeout_atomic(master->regs + RESET_CTRL, status,
 458                                           !status, 10, 1000000);
 459         } else {
 460                 list_del_init(&xfer->node);
 461         }
 462 }
 463
 464 static void dw_i3c_master_dequeue_xfer(struct dw_i3c_master *master,
 465                                        struct dw_i3c_xfer *xfer)
 466 {
 467         unsigned long flags;
 468
 469         spin_lock_irqsave(&master->xferqueue.lock, flags);
 470         dw_i3c_master_dequeue_xfer_locked(master, xfer);
 471         spin_unlock_irqrestore(&master->xferqueue.lock, flags);
 472 }
 473
 474 static void dw_i3c_master_end_xfer_locked(struct dw_i3c_master *master, u32 isr)
 475 {
 476         struct dw_i3c_xfer *xfer = master->xferqueue.cur;
 477         int i, ret = 0;
 478         u32 nresp;
 479
 480         if (!xfer)
 481                 return;
 482
 483         nresp = readl(master->regs + QUEUE_STATUS_LEVEL);
 484         nresp = QUEUE_STATUS_LEVEL_RESP(nresp);
 485
 486         for (i = 0; i < nresp; i++) {
 487                 struct dw_i3c_cmd *cmd;
 488                 u32 resp;
 489
 490                 resp = readl(master->regs + RESPONSE_QUEUE_PORT);
 491
 492                 cmd = &xfer->cmds[RESPONSE_PORT_TID(resp)];
 493                 cmd->rx_len = RESPONSE_PORT_DATA_LEN(resp);
 494                 cmd->error = RESPONSE_PORT_ERR_STATUS(resp);
 495                 if (cmd->rx_len && !cmd->error)
 496                         dw_i3c_master_read_rx_fifo(master, cmd->rx_buf,
 497                                                    cmd->rx_len);
 498         }
 499
 500         for (i = 0; i < nresp; i++) {
 501                 switch (xfer->cmds[i].error) {
 502                 case RESPONSE_NO_ERROR:
 503                         break;
 504                 case RESPONSE_ERROR_PARITY:
 505                 case RESPONSE_ERROR_IBA_NACK:
 506                 case RESPONSE_ERROR_TRANSF_ABORT:
 507                 case RESPONSE_ERROR_CRC:
 508                 case RESPONSE_ERROR_FRAME:
 509                         ret = -EIO;
 510                         break;
 511                 case RESPONSE_ERROR_OVER_UNDER_FLOW:
 512                         ret = -ENOSPC;
 513                         break;
 514                 case RESPONSE_ERROR_I2C_W_NACK_ERR:
 515                 case RESPONSE_ERROR_ADDRESS_NACK:
 516                 default:
 517                         ret = -EINVAL;
 518                         break;
 519                 }
 520         }
 521
 522         xfer->ret = ret;
 523         complete(&xfer->comp);
 524
 525         if (ret < 0) {
 526                 dw_i3c_master_dequeue_xfer_locked(master, xfer);
 527                 writel(readl(master->regs + DEVICE_CTRL) | DEV_CTRL_RESUME,
 528                        master->regs + DEVICE_CTRL);
 529         }
 530
 531         xfer = list_first_entry_or_null(&master->xferqueue.list,
 532                                         struct dw_i3c_xfer,
 533                                         node);
 534         if (xfer)
 535                 list_del_init(&xfer->node);
 536
 537         master->xferqueue.cur = xfer;
 538         dw_i3c_master_start_xfer_locked(master);
 539 }
 540
 541 static void dw_i3c_master_set_intr_regs(struct dw_i3c_master *master)
 542 {
 543         u32 thld_ctrl;
 544
 545         thld_ctrl = readl(master->regs + QUEUE_THLD_CTRL);
 546         thld_ctrl &= ~(QUEUE_THLD_CTRL_RESP_BUF_MASK |
 547                        QUEUE_THLD_CTRL_IBI_STAT_MASK |
 548                        QUEUE_THLD_CTRL_IBI_DATA_MASK);
 549         thld_ctrl |= QUEUE_THLD_CTRL_IBI_STAT(1) |
 550                 QUEUE_THLD_CTRL_IBI_DATA(31);
 551         writel(thld_ctrl, master->regs + QUEUE_THLD_CTRL);
 552
 553         thld_ctrl = readl(master->regs + DATA_BUFFER_THLD_CTRL);
 554         thld_ctrl &= ~DATA_BUFFER_THLD_CTRL_RX_BUF;
 555         writel(thld_ctrl, master->regs + DATA_BUFFER_THLD_CTRL);
 556
 557         writel(INTR_ALL, master->regs + INTR_STATUS);
 558         writel(INTR_MASTER_MASK, master->regs + INTR_STATUS_EN);
 559         writel(INTR_MASTER_MASK, master->regs + INTR_SIGNAL_EN);
 560
 561         master->sir_rej_mask = IBI_REQ_REJECT_ALL;
 562         writel(master->sir_rej_mask, master->regs + IBI_SIR_REQ_REJECT);
 563
 564         writel(IBI_REQ_REJECT_ALL, master->regs + IBI_MR_REQ_REJECT);
 565 }
 566
 567 static int dw_i3c_clk_cfg(struct dw_i3c_master *master)
 568 {
 569         unsigned long core_rate, core_period;
 570         u32 scl_timing;
 571         u8 hcnt, lcnt;
 572
 573         core_rate = clk_get_rate(master->core_clk);
 574         if (!core_rate)
 575                 return -EINVAL;
 576
 577         core_period = DIV_ROUND_UP(1000000000, core_rate);
 578
 579         hcnt = DIV_ROUND_UP(I3C_BUS_THIGH_MAX_NS, core_period) - 1;
 580         if (hcnt < SCL_I3C_TIMING_CNT_MIN)
 581                 hcnt = SCL_I3C_TIMING_CNT_MIN;
 582
 583         lcnt = DIV_ROUND_UP(core_rate, master->base.bus.scl_rate.i3c) - hcnt;
 584         if (lcnt < SCL_I3C_TIMING_CNT_MIN)
 585                 lcnt = SCL_I3C_TIMING_CNT_MIN;
 586
 587         scl_timing = SCL_I3C_TIMING_HCNT(hcnt) | SCL_I3C_TIMING_LCNT(lcnt);
 588         writel(scl_timing, master->regs + SCL_I3C_PP_TIMING);
 589         master->i3c_pp_timing = scl_timing;
 590
 591         /*
 592          * In pure i3c mode, MST_FREE represents tCAS. In shared mode, this
 593          * will be set up by dw_i2c_clk_cfg as tLOW.
 594          */
 595         if (master->base.bus.mode == I3C_BUS_MODE_PURE) {
 596                 writel(BUS_I3C_MST_FREE(lcnt), master->regs + BUS_FREE_TIMING);
 597                 master->bus_free_timing = BUS_I3C_MST_FREE(lcnt);
 598         }
 599
 600         lcnt = max_t(u8,
 601                      DIV_ROUND_UP(I3C_BUS_TLOW_OD_MIN_NS, core_period), lcnt);
 602         scl_timing = SCL_I3C_TIMING_HCNT(hcnt) | SCL_I3C_TIMING_LCNT(lcnt);
 603         writel(scl_timing, master->regs + SCL_I3C_OD_TIMING);
 604         master->i3c_od_timing = scl_timing;
 605
 606         lcnt = DIV_ROUND_UP(core_rate, I3C_BUS_SDR1_SCL_RATE) - hcnt;
 607         scl_timing = SCL_EXT_LCNT_1(lcnt);
 608         lcnt = DIV_ROUND_UP(core_rate, I3C_BUS_SDR2_SCL_RATE) - hcnt;
 609         scl_timing |= SCL_EXT_LCNT_2(lcnt);
 610         lcnt = DIV_ROUND_UP(core_rate, I3C_BUS_SDR3_SCL_RATE) - hcnt;
 611         scl_timing |= SCL_EXT_LCNT_3(lcnt);
 612         lcnt = DIV_ROUND_UP(core_rate, I3C_BUS_SDR4_SCL_RATE) - hcnt;
 613         scl_timing |= SCL_EXT_LCNT_4(lcnt);
 614         writel(scl_timing, master->regs + SCL_EXT_LCNT_TIMING);
 615         master->ext_lcnt_timing = scl_timing;
 616
 617         return 0;
 618 }
 619
 620 static int dw_i2c_clk_cfg(struct dw_i3c_master *master)
 621 {
 622         unsigned long core_rate, core_period;
 623         u16 hcnt, lcnt;
 624         u32 scl_timing;
 625
 626         core_rate = clk_get_rate(master->core_clk);
 627         if (!core_rate)
 628                 return -EINVAL;
 629
 630         core_period = DIV_ROUND_UP(1000000000, core_rate);
 631
 632         lcnt = DIV_ROUND_UP(I3C_BUS_I2C_FMP_TLOW_MIN_NS, core_period);
 633         hcnt = DIV_ROUND_UP(core_rate, I3C_BUS_I2C_FM_PLUS_SCL_RATE) - lcnt;
 634         scl_timing = SCL_I2C_FMP_TIMING_HCNT(hcnt) |
 635                      SCL_I2C_FMP_TIMING_LCNT(lcnt);
 636         writel(scl_timing, master->regs + SCL_I2C_FMP_TIMING);
 637         master->i2c_fmp_timing = scl_timing;
 638
 639         lcnt = DIV_ROUND_UP(I3C_BUS_I2C_FM_TLOW_MIN_NS, core_period);
 640         hcnt = DIV_ROUND_UP(core_rate, I3C_BUS_I2C_FM_SCL_RATE) - lcnt;
 641         scl_timing = SCL_I2C_FM_TIMING_HCNT(hcnt) |
 642                      SCL_I2C_FM_TIMING_LCNT(lcnt);
 643         writel(scl_timing, master->regs + SCL_I2C_FM_TIMING);
 644         master->i2c_fm_timing = scl_timing;
 645
 646         writel(BUS_I3C_MST_FREE(lcnt), master->regs + BUS_FREE_TIMING);
 647         master->bus_free_timing = BUS_I3C_MST_FREE(lcnt);
 648
 649         writel(readl(master->regs + DEVICE_CTRL) | DEV_CTRL_I2C_SLAVE_PRESENT,
 650                master->regs + DEVICE_CTRL);
 651         master->i2c_slv_prsnt = true;
 652
 653         return 0;
 654 }
 655
 656 static int dw_i3c_master_bus_init(struct i3c_master_controller *m)
 657 {
 658         struct dw_i3c_master *master = to_dw_i3c_master(m);
 659         struct i3c_bus *bus = i3c_master_get_bus(m);
 660         struct i3c_device_info info = { };
 661         int ret;
 662
 663         ret = pm_runtime_resume_and_get(master->dev);
 664         if (ret < 0) {
 665                 dev_err(master->dev,
 666                         "<%s> cannot resume i3c bus master, err: %d\n",
 667                         __func__, ret);
 668                 return ret;
 669         }
 670
 671         ret = master->platform_ops->init(master);
 672         if (ret)
 673                 goto rpm_out;
 674
 675         switch (bus->mode) {
 676         case I3C_BUS_MODE_MIXED_FAST:
 677         case I3C_BUS_MODE_MIXED_LIMITED:
 678                 ret = dw_i2c_clk_cfg(master);
 679                 if (ret)
 680                         goto rpm_out;
 681                 fallthrough;
 682         case I3C_BUS_MODE_PURE:
 683                 ret = dw_i3c_clk_cfg(master);
 684                 if (ret)
 685                         goto rpm_out;
 686                 break;
 687         default:
 688                 ret = -EINVAL;
 689                 goto rpm_out;
 690         }
 691
 692         ret = i3c_master_get_free_addr(m, 0);
 693         if (ret < 0)
 694                 goto rpm_out;
 695
 696         writel(DEV_ADDR_DYNAMIC_ADDR_VALID | DEV_ADDR_DYNAMIC(ret),
 697                master->regs + DEVICE_ADDR);
 698         master->dev_addr = ret;
 699         memset(&info, 0, sizeof(info));
 700         info.dyn_addr = ret;
 701
 702         ret = i3c_master_set_info(&master->base, &info);
 703         if (ret)
 704                 goto rpm_out;
 705
 706         dw_i3c_master_set_intr_regs(master);
 707         dw_i3c_master_enable(master);
 708
 709 rpm_out:
 710         pm_runtime_mark_last_busy(master->dev);
 711         pm_runtime_put_autosuspend(master->dev);
 712         return ret;
 713 }
 714
 715 static void dw_i3c_master_bus_cleanup(struct i3c_master_controller *m)
 716 {
 717         struct dw_i3c_master *master = to_dw_i3c_master(m);
 718
 719         dw_i3c_master_disable(master);
 720 }
 721
 722 static int dw_i3c_ccc_set(struct dw_i3c_master *master,
 723                           struct i3c_ccc_cmd *ccc)
 724 {
 725         struct dw_i3c_xfer *xfer;
 726         struct dw_i3c_cmd *cmd;
 727         int ret, pos = 0;
 728
 729         if (ccc->id & I3C_CCC_DIRECT) {
 730                 pos = dw_i3c_master_get_addr_pos(master, ccc->dests[0].addr);
 731                 if (pos < 0)
 732                         return pos;
 733         }
 734
 735         xfer = dw_i3c_master_alloc_xfer(master, 1);
 736         if (!xfer)
 737                 return -ENOMEM;
 738
 739         cmd = xfer->cmds;
 740         cmd->tx_buf = ccc->dests[0].payload.data;
 741         cmd->tx_len = ccc->dests[0].payload.len;
 742
 743         cmd->cmd_hi = COMMAND_PORT_ARG_DATA_LEN(ccc->dests[0].payload.len) |
 744                       COMMAND_PORT_TRANSFER_ARG;
 745
 746         cmd->cmd_lo = COMMAND_PORT_CP |
 747                       COMMAND_PORT_DEV_INDEX(pos) |
 748                       COMMAND_PORT_CMD(ccc->id) |
 749                       COMMAND_PORT_TOC |
 750                       COMMAND_PORT_ROC;
 751
 752         dw_i3c_master_enqueue_xfer(master, xfer);
 753         if (!wait_for_completion_timeout(&xfer->comp, XFER_TIMEOUT))
 754                 dw_i3c_master_dequeue_xfer(master, xfer);
 755
 756         ret = xfer->ret;
 757         if (xfer->cmds[0].error == RESPONSE_ERROR_IBA_NACK)
 758                 ccc->err = I3C_ERROR_M2;
 759
 760         dw_i3c_master_free_xfer(xfer);
 761
 762         return ret;
 763 }
 764
 765 static int dw_i3c_ccc_get(struct dw_i3c_master *master, struct i3c_ccc_cmd *ccc)
 766 {
 767         struct dw_i3c_xfer *xfer;
 768         struct dw_i3c_cmd *cmd;
 769         int ret, pos;
 770
 771         pos = dw_i3c_master_get_addr_pos(master, ccc->dests[0].addr);
 772         if (pos < 0)
 773                 return pos;
 774
 775         xfer = dw_i3c_master_alloc_xfer(master, 1);
 776         if (!xfer)
 777                 return -ENOMEM;
 778
 779         cmd = xfer->cmds;
 780         cmd->rx_buf = ccc->dests[0].payload.data;
 781         cmd->rx_len = ccc->dests[0].payload.len;
 782
 783         cmd->cmd_hi = COMMAND_PORT_ARG_DATA_LEN(ccc->dests[0].payload.len) |
 784                       COMMAND_PORT_TRANSFER_ARG;
 785
 786         cmd->cmd_lo = COMMAND_PORT_READ_TRANSFER |
 787                       COMMAND_PORT_CP |
 788                       COMMAND_PORT_DEV_INDEX(pos) |
 789                       COMMAND_PORT_CMD(ccc->id) |
 790                       COMMAND_PORT_TOC |
 791                       COMMAND_PORT_ROC;
 792
 793         dw_i3c_master_enqueue_xfer(master, xfer);
 794         if (!wait_for_completion_timeout(&xfer->comp, XFER_TIMEOUT))
 795                 dw_i3c_master_dequeue_xfer(master, xfer);
 796
 797         ret = xfer->ret;
 798         if (xfer->cmds[0].error == RESPONSE_ERROR_IBA_NACK)
 799                 ccc->err = I3C_ERROR_M2;
 800         dw_i3c_master_free_xfer(xfer);
 801
 802         return ret;
 803 }
 804
 805 static void amd_configure_od_pp_quirk(struct dw_i3c_master *master)
 806 {
 807         master->i3c_od_timing = AMD_I3C_OD_TIMING;
 808         master->i3c_pp_timing = AMD_I3C_PP_TIMING;
 809 }
 810
 811 static int dw_i3c_master_send_ccc_cmd(struct i3c_master_controller *m,
 812                                       struct i3c_ccc_cmd *ccc)
 813 {
 814         struct dw_i3c_master *master = to_dw_i3c_master(m);
 815         int ret = 0;
 816
 817         if (ccc->id == I3C_CCC_ENTDAA)
 818                 return -EINVAL;
 819
 820         /* AMD platform specific OD and PP timings */
 821         if (master->quirks & AMD_I3C_OD_PP_TIMING) {
 822                 amd_configure_od_pp_quirk(master);
 823                 writel(master->i3c_pp_timing, master->regs + SCL_I3C_PP_TIMING);
 824                 writel(master->i3c_od_timing, master->regs + SCL_I3C_OD_TIMING);
 825         }
 826
 827         ret = pm_runtime_resume_and_get(master->dev);
 828         if (ret < 0) {
 829                 dev_err(master->dev,
 830                         "<%s> cannot resume i3c bus master, err: %d\n",
 831                         __func__, ret);
 832                 return ret;
 833         }
 834
 835         if (ccc->rnw)
 836                 ret = dw_i3c_ccc_get(master, ccc);
 837         else
 838                 ret = dw_i3c_ccc_set(master, ccc);
 839
 840         pm_runtime_mark_last_busy(master->dev);
 841         pm_runtime_put_autosuspend(master->dev);
 842         return ret;
 843 }
 844
 845 static int dw_i3c_master_daa(struct i3c_master_controller *m)
 846 {
 847         struct dw_i3c_master *master = to_dw_i3c_master(m);
 848         struct dw_i3c_xfer *xfer;
 849         struct dw_i3c_cmd *cmd;
 850         u32 olddevs, newdevs;
 851         u8 p, last_addr = 0;
 852         int ret, pos;
 853
 854         ret = pm_runtime_resume_and_get(master->dev);
 855         if (ret < 0) {
 856                 dev_err(master->dev,
 857                         "<%s> cannot resume i3c bus master, err: %d\n",
 858                         __func__, ret);
 859                 return ret;
 860         }
 861
 862         olddevs = ~(master->free_pos);
 863
 864         /* Prepare DAT before launching DAA. */
 865         for (pos = 0; pos < master->maxdevs; pos++) {
 866                 if (olddevs & BIT(pos))
 867                         continue;
 868
 869                 ret = i3c_master_get_free_addr(m, last_addr + 1);
 870                 if (ret < 0) {
 871                         ret = -ENOSPC;
 872                         goto rpm_out;
 873                 }
 874
 875                 master->devs[pos].addr = ret;
 876                 p = even_parity(ret);
 877                 last_addr = ret;
 878                 ret |= (p << 7);
 879
 880                 writel(DEV_ADDR_TABLE_DYNAMIC_ADDR(ret),
 881                        master->regs +
 882                        DEV_ADDR_TABLE_LOC(master->datstartaddr, pos));
 883
 884                 ret = 0;
 885         }
 886
 887         xfer = dw_i3c_master_alloc_xfer(master, 1);
 888         if (!xfer) {
 889                 ret = -ENOMEM;
 890                 goto rpm_out;
 891         }
 892
 893         pos = dw_i3c_master_get_free_pos(master);
 894         if (pos < 0) {
 895                 dw_i3c_master_free_xfer(xfer);
 896                 ret = pos;
 897                 goto rpm_out;
 898         }
 899         cmd = &xfer->cmds[0];
 900         cmd->cmd_hi = 0x1;
 901         cmd->cmd_lo = COMMAND_PORT_DEV_COUNT(master->maxdevs - pos) |
 902                       COMMAND_PORT_DEV_INDEX(pos) |
 903                       COMMAND_PORT_CMD(I3C_CCC_ENTDAA) |
 904                       COMMAND_PORT_ADDR_ASSGN_CMD |
 905                       COMMAND_PORT_TOC |
 906                       COMMAND_PORT_ROC;
 907
 908         dw_i3c_master_enqueue_xfer(master, xfer);
 909         if (!wait_for_completion_timeout(&xfer->comp, XFER_TIMEOUT))
 910                 dw_i3c_master_dequeue_xfer(master, xfer);
 911
 912         newdevs = GENMASK(master->maxdevs - cmd->rx_len - 1, 0);
 913         newdevs &= ~olddevs;
 914
 915         for (pos = 0; pos < master->maxdevs; pos++) {
 916                 if (newdevs & BIT(pos))
 917                         i3c_master_add_i3c_dev_locked(m, master->devs[pos].addr);
 918         }
 919
 920         dw_i3c_master_free_xfer(xfer);
 921
 922 rpm_out:
 923         pm_runtime_mark_last_busy(master->dev);
 924         pm_runtime_put_autosuspend(master->dev);
 925         return ret;
 926 }
 927
 928 static int dw_i3c_master_priv_xfers(struct i3c_dev_desc *dev,
 929                                     struct i3c_priv_xfer *i3c_xfers,
 930                                     int i3c_nxfers)
 931 {
 932         struct dw_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev);
 933         struct i3c_master_controller *m = i3c_dev_get_master(dev);
 934         struct dw_i3c_master *master = to_dw_i3c_master(m);
 935         unsigned int nrxwords = 0, ntxwords = 0;
 936         struct dw_i3c_xfer *xfer;
 937         int i, ret = 0;
 938
 939         if (!i3c_nxfers)
 940                 return 0;
 941
 942         if (i3c_nxfers > master->caps.cmdfifodepth)
 943                 return -ENOTSUPP;
 944
 945         for (i = 0; i < i3c_nxfers; i++) {
 946                 if (i3c_xfers[i].rnw)
 947                         nrxwords += DIV_ROUND_UP(i3c_xfers[i].len, 4);
 948                 else
 949                         ntxwords += DIV_ROUND_UP(i3c_xfers[i].len, 4);
 950         }
 951
 952         if (ntxwords > master->caps.datafifodepth ||
 953             nrxwords > master->caps.datafifodepth)
 954                 return -ENOTSUPP;
 955
 956         xfer = dw_i3c_master_alloc_xfer(master, i3c_nxfers);
 957         if (!xfer)
 958                 return -ENOMEM;
 959
 960         ret = pm_runtime_resume_and_get(master->dev);
 961         if (ret < 0) {
 962                 dev_err(master->dev,
 963                         "<%s> cannot resume i3c bus master, err: %d\n",
 964                         __func__, ret);
 965                 return ret;
 966         }
 967
 968         for (i = 0; i < i3c_nxfers; i++) {
 969                 struct dw_i3c_cmd *cmd = &xfer->cmds[i];
 970
 971                 cmd->cmd_hi = COMMAND_PORT_ARG_DATA_LEN(i3c_xfers[i].len) |
 972                         COMMAND_PORT_TRANSFER_ARG;
 973
 974                 if (i3c_xfers[i].rnw) {
 975                         cmd->rx_buf = i3c_xfers[i].data.in;
 976                         cmd->rx_len = i3c_xfers[i].len;
 977                         cmd->cmd_lo = COMMAND_PORT_READ_TRANSFER |
 978                                       COMMAND_PORT_SPEED(dev->info.max_read_ds);
 979
 980                 } else {
 981                         cmd->tx_buf = i3c_xfers[i].data.out;
 982                         cmd->tx_len = i3c_xfers[i].len;
 983                         cmd->cmd_lo =
 984                                 COMMAND_PORT_SPEED(dev->info.max_write_ds);
 985                 }
 986
 987                 cmd->cmd_lo |= COMMAND_PORT_TID(i) |
 988                                COMMAND_PORT_DEV_INDEX(data->index) |
 989                                COMMAND_PORT_ROC;
 990
 991                 if (i == (i3c_nxfers - 1))
 992                         cmd->cmd_lo |= COMMAND_PORT_TOC;
 993         }
 994
 995         dw_i3c_master_enqueue_xfer(master, xfer);
 996         if (!wait_for_completion_timeout(&xfer->comp, XFER_TIMEOUT))
 997                 dw_i3c_master_dequeue_xfer(master, xfer);
 998
 999         for (i = 0; i < i3c_nxfers; i++) {
1000                 struct dw_i3c_cmd *cmd = &xfer->cmds[i];
1001
1002                 if (i3c_xfers[i].rnw)
1003                         i3c_xfers[i].len = cmd->rx_len;
1004         }
1005
1006         ret = xfer->ret;
1007         dw_i3c_master_free_xfer(xfer);
1008
1009         pm_runtime_mark_last_busy(master->dev);
1010         pm_runtime_put_autosuspend(master->dev);
1011         return ret;
1012 }
1013
1014 static int dw_i3c_master_reattach_i3c_dev(struct i3c_dev_desc *dev,
1015                                           u8 old_dyn_addr)
1016 {
1017         struct dw_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev);
1018         struct i3c_master_controller *m = i3c_dev_get_master(dev);
1019         struct dw_i3c_master *master = to_dw_i3c_master(m);
1020         int pos;
1021
1022         pos = dw_i3c_master_get_free_pos(master);
1023
1024         if (data->index > pos && pos > 0) {
1025                 writel(0,
1026                        master->regs +
1027                        DEV_ADDR_TABLE_LOC(master->datstartaddr, data->index));
1028
1029                 master->devs[data->index].addr = 0;
1030                 master->free_pos |= BIT(data->index);
1031
1032                 data->index = pos;
1033                 master->devs[pos].addr = dev->info.dyn_addr;
1034                 master->free_pos &= ~BIT(pos);
1035         }
1036
1037         writel(DEV_ADDR_TABLE_DYNAMIC_ADDR(dev->info.dyn_addr),
1038                master->regs +
1039                DEV_ADDR_TABLE_LOC(master->datstartaddr, data->index));
1040
1041         master->devs[data->index].addr = dev->info.dyn_addr;
1042
1043         return 0;
1044 }
1045
1046 static int dw_i3c_master_attach_i3c_dev(struct i3c_dev_desc *dev)
1047 {
1048         struct i3c_master_controller *m = i3c_dev_get_master(dev);
1049         struct dw_i3c_master *master = to_dw_i3c_master(m);
1050         struct dw_i3c_i2c_dev_data *data;
1051         int pos;
1052
1053         pos = dw_i3c_master_get_free_pos(master);
1054         if (pos < 0)
1055                 return pos;
1056
1057         data = kzalloc(sizeof(*data), GFP_KERNEL);
1058         if (!data)
1059                 return -ENOMEM;
1060
1061         data->index = pos;
1062         master->devs[pos].addr = dev->info.dyn_addr ? : dev->info.static_addr;
1063         master->free_pos &= ~BIT(pos);
1064         i3c_dev_set_master_data(dev, data);
1065
1066         writel(DEV_ADDR_TABLE_DYNAMIC_ADDR(master->devs[pos].addr),
1067                master->regs +
1068                DEV_ADDR_TABLE_LOC(master->datstartaddr, data->index));
1069
1070         return 0;
1071 }
1072
1073 static void dw_i3c_master_detach_i3c_dev(struct i3c_dev_desc *dev)
1074 {
1075         struct dw_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev);
1076         struct i3c_master_controller *m = i3c_dev_get_master(dev);
1077         struct dw_i3c_master *master = to_dw_i3c_master(m);
1078
1079         writel(0,
1080                master->regs +
1081                DEV_ADDR_TABLE_LOC(master->datstartaddr, data->index));
1082
1083         i3c_dev_set_master_data(dev, NULL);
1084         master->devs[data->index].addr = 0;
1085         master->free_pos |= BIT(data->index);
1086         kfree(data);
1087 }
1088
1089 static int dw_i3c_master_i2c_xfers(struct i2c_dev_desc *dev,
1090                                    const struct i2c_msg *i2c_xfers,
1091                                    int i2c_nxfers)
1092 {
1093         struct dw_i3c_i2c_dev_data *data = i2c_dev_get_master_data(dev);
1094         struct i3c_master_controller *m = i2c_dev_get_master(dev);
1095         struct dw_i3c_master *master = to_dw_i3c_master(m);
1096         unsigned int nrxwords = 0, ntxwords = 0;
1097         struct dw_i3c_xfer *xfer;
1098         int i, ret = 0;
1099
1100         if (!i2c_nxfers)
1101                 return 0;
1102
1103         if (i2c_nxfers > master->caps.cmdfifodepth)
1104                 return -ENOTSUPP;
1105
1106         for (i = 0; i < i2c_nxfers; i++) {
1107                 if (i2c_xfers[i].flags & I2C_M_RD)
1108                         nrxwords += DIV_ROUND_UP(i2c_xfers[i].len, 4);
1109                 else
1110                         ntxwords += DIV_ROUND_UP(i2c_xfers[i].len, 4);
1111         }
1112
1113         if (ntxwords > master->caps.datafifodepth ||
1114             nrxwords > master->caps.datafifodepth)
1115                 return -ENOTSUPP;
1116
1117         xfer = dw_i3c_master_alloc_xfer(master, i2c_nxfers);
1118         if (!xfer)
1119                 return -ENOMEM;
1120
1121         ret = pm_runtime_resume_and_get(master->dev);
1122         if (ret < 0) {
1123                 dev_err(master->dev,
1124                         "<%s> cannot resume i3c bus master, err: %d\n",
1125                         __func__, ret);
1126                 return ret;
1127         }
1128
1129         for (i = 0; i < i2c_nxfers; i++) {
1130                 struct dw_i3c_cmd *cmd = &xfer->cmds[i];
1131
1132                 cmd->cmd_hi = COMMAND_PORT_ARG_DATA_LEN(i2c_xfers[i].len) |
1133                         COMMAND_PORT_TRANSFER_ARG;
1134
1135                 cmd->cmd_lo = COMMAND_PORT_TID(i) |
1136                               COMMAND_PORT_DEV_INDEX(data->index) |
1137                               COMMAND_PORT_ROC;
1138
1139                 if (i2c_xfers[i].flags & I2C_M_RD) {
1140                         cmd->cmd_lo |= COMMAND_PORT_READ_TRANSFER;
1141                         cmd->rx_buf = i2c_xfers[i].buf;
1142                         cmd->rx_len = i2c_xfers[i].len;
1143                 } else {
1144                         cmd->tx_buf = i2c_xfers[i].buf;
1145                         cmd->tx_len = i2c_xfers[i].len;
1146                 }
1147
1148                 if (i == (i2c_nxfers - 1))
1149                         cmd->cmd_lo |= COMMAND_PORT_TOC;
1150         }
1151
1152         dw_i3c_master_enqueue_xfer(master, xfer);
1153         if (!wait_for_completion_timeout(&xfer->comp, XFER_TIMEOUT))
1154                 dw_i3c_master_dequeue_xfer(master, xfer);
1155
1156         ret = xfer->ret;
1157         dw_i3c_master_free_xfer(xfer);
1158
1159         pm_runtime_mark_last_busy(master->dev);
1160         pm_runtime_put_autosuspend(master->dev);
1161         return ret;
1162 }
1163
1164 static int dw_i3c_master_attach_i2c_dev(struct i2c_dev_desc *dev)
1165 {
1166         struct i3c_master_controller *m = i2c_dev_get_master(dev);
1167         struct dw_i3c_master *master = to_dw_i3c_master(m);
1168         struct dw_i3c_i2c_dev_data *data;
1169         int pos;
1170
1171         pos = dw_i3c_master_get_free_pos(master);
1172         if (pos < 0)
1173                 return pos;
1174
1175         data = kzalloc(sizeof(*data), GFP_KERNEL);
1176         if (!data)
1177                 return -ENOMEM;
1178
1179         data->index = pos;
1180         master->devs[pos].addr = dev->addr;
1181         master->devs[pos].is_i2c_addr = true;
1182         master->free_pos &= ~BIT(pos);
1183         i2c_dev_set_master_data(dev, data);
1184
1185         writel(DEV_ADDR_TABLE_LEGACY_I2C_DEV |
1186                DEV_ADDR_TABLE_STATIC_ADDR(dev->addr),
1187                master->regs +
1188                DEV_ADDR_TABLE_LOC(master->datstartaddr, data->index));
1189
1190         return 0;
1191 }
1192
1193 static void dw_i3c_master_detach_i2c_dev(struct i2c_dev_desc *dev)
1194 {
1195         struct dw_i3c_i2c_dev_data *data = i2c_dev_get_master_data(dev);
1196         struct i3c_master_controller *m = i2c_dev_get_master(dev);
1197         struct dw_i3c_master *master = to_dw_i3c_master(m);
1198
1199         writel(0,
1200                master->regs +
1201                DEV_ADDR_TABLE_LOC(master->datstartaddr, data->index));
1202
1203         i2c_dev_set_master_data(dev, NULL);
1204         master->devs[data->index].addr = 0;
1205         master->free_pos |= BIT(data->index);
1206         kfree(data);
1207 }
1208
1209 static int dw_i3c_master_request_ibi(struct i3c_dev_desc *dev,
1210                                      const struct i3c_ibi_setup *req)
1211 {
1212         struct dw_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev);
1213         struct i3c_master_controller *m = i3c_dev_get_master(dev);
1214         struct dw_i3c_master *master = to_dw_i3c_master(m);
1215         unsigned long flags;
1216
1217         data->ibi_pool = i3c_generic_ibi_alloc_pool(dev, req);
1218         if (IS_ERR(data->ibi_pool))
1219                 return PTR_ERR(data->ibi_pool);
1220
1221         spin_lock_irqsave(&master->devs_lock, flags);
1222         master->devs[data->index].ibi_dev = dev;
1223         spin_unlock_irqrestore(&master->devs_lock, flags);
1224
1225         return 0;
1226 }
1227
1228 static void dw_i3c_master_free_ibi(struct i3c_dev_desc *dev)
1229 {
1230         struct dw_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev);
1231         struct i3c_master_controller *m = i3c_dev_get_master(dev);
1232         struct dw_i3c_master *master = to_dw_i3c_master(m);
1233         unsigned long flags;
1234
1235         spin_lock_irqsave(&master->devs_lock, flags);
1236         master->devs[data->index].ibi_dev = NULL;
1237         spin_unlock_irqrestore(&master->devs_lock, flags);
1238
1239         i3c_generic_ibi_free_pool(data->ibi_pool);
1240         data->ibi_pool = NULL;
1241 }
1242
1243 static void dw_i3c_master_enable_sir_signal(struct dw_i3c_master *master, bool enable)
1244 {
1245         u32 reg;
1246
1247         reg = readl(master->regs + INTR_STATUS_EN);
1248         reg &= ~INTR_IBI_THLD_STAT;
1249         if (enable)
1250                 reg |= INTR_IBI_THLD_STAT;
1251         writel(reg, master->regs + INTR_STATUS_EN);
1252
1253         reg = readl(master->regs + INTR_SIGNAL_EN);
1254         reg &= ~INTR_IBI_THLD_STAT;
1255         if (enable)
1256                 reg |= INTR_IBI_THLD_STAT;
1257         writel(reg, master->regs + INTR_SIGNAL_EN);
1258 }
1259
1260 static void dw_i3c_master_set_sir_enabled(struct dw_i3c_master *master,
1261                                           struct i3c_dev_desc *dev,
1262                                           u8 idx, bool enable)
1263 {
1264         unsigned long flags;
1265         u32 dat_entry, reg;
1266         bool global;
1267
1268         dat_entry = DEV_ADDR_TABLE_LOC(master->datstartaddr, idx);
1269
1270         spin_lock_irqsave(&master->devs_lock, flags);
1271         reg = readl(master->regs + dat_entry);
1272         if (enable) {
1273                 reg &= ~DEV_ADDR_TABLE_SIR_REJECT;
1274                 if (dev->info.bcr & I3C_BCR_IBI_PAYLOAD)
1275                         reg |= DEV_ADDR_TABLE_IBI_MDB;
1276         } else {
1277                 reg |= DEV_ADDR_TABLE_SIR_REJECT;
1278         }
1279         master->platform_ops->set_dat_ibi(master, dev, enable, &reg);
1280         writel(reg, master->regs + dat_entry);
1281
1282         if (enable) {
1283                 global = (master->sir_rej_mask == IBI_REQ_REJECT_ALL);
1284                 master->sir_rej_mask &= ~BIT(idx);
1285         } else {
1286                 bool hj_rejected = !!(readl(master->regs + DEVICE_CTRL) & DEV_CTRL_HOT_JOIN_NACK);
1287
1288                 master->sir_rej_mask |= BIT(idx);
1289                 global = (master->sir_rej_mask == IBI_REQ_REJECT_ALL) && hj_rejected;
1290         }
1291         writel(master->sir_rej_mask, master->regs + IBI_SIR_REQ_REJECT);
1292
1293         if (global)
1294                 dw_i3c_master_enable_sir_signal(master, enable);
1295
1296
1297         spin_unlock_irqrestore(&master->devs_lock, flags);
1298 }
1299
1300 static int dw_i3c_master_enable_hotjoin(struct i3c_master_controller *m)
1301 {
1302         struct dw_i3c_master *master = to_dw_i3c_master(m);
1303         int ret;
1304
1305         ret = pm_runtime_resume_and_get(master->dev);
1306         if (ret < 0) {
1307                 dev_err(master->dev,
1308                         "<%s> cannot resume i3c bus master, err: %d\n",
1309                         __func__, ret);
1310                 return ret;
1311         }
1312
1313         dw_i3c_master_enable_sir_signal(master, true);
1314         writel(readl(master->regs + DEVICE_CTRL) & ~DEV_CTRL_HOT_JOIN_NACK,
1315                master->regs + DEVICE_CTRL);
1316
1317         return 0;
1318 }
1319
1320 static int dw_i3c_master_disable_hotjoin(struct i3c_master_controller *m)
1321 {
1322         struct dw_i3c_master *master = to_dw_i3c_master(m);
1323
1324         writel(readl(master->regs + DEVICE_CTRL) | DEV_CTRL_HOT_JOIN_NACK,
1325                master->regs + DEVICE_CTRL);
1326
1327         pm_runtime_mark_last_busy(master->dev);
1328         pm_runtime_put_autosuspend(master->dev);
1329         return 0;
1330 }
1331
1332 static int dw_i3c_master_enable_ibi(struct i3c_dev_desc *dev)
1333 {
1334         struct dw_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev);
1335         struct i3c_master_controller *m = i3c_dev_get_master(dev);
1336         struct dw_i3c_master *master = to_dw_i3c_master(m);
1337         int rc;
1338
1339         rc = pm_runtime_resume_and_get(master->dev);
1340         if (rc < 0) {
1341                 dev_err(master->dev,
1342                         "<%s> cannot resume i3c bus master, err: %d\n",
1343                         __func__, rc);
1344                 return rc;
1345         }
1346
1347         dw_i3c_master_set_sir_enabled(master, dev, data->index, true);
1348
1349         rc = i3c_master_enec_locked(m, dev->info.dyn_addr, I3C_CCC_EVENT_SIR);
1350
1351         if (rc) {
1352                 dw_i3c_master_set_sir_enabled(master, dev, data->index, false);
1353                 pm_runtime_mark_last_busy(master->dev);
1354                 pm_runtime_put_autosuspend(master->dev);
1355         }
1356
1357         return rc;
1358 }
1359
1360 static int dw_i3c_master_disable_ibi(struct i3c_dev_desc *dev)
1361 {
1362         struct dw_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev);
1363         struct i3c_master_controller *m = i3c_dev_get_master(dev);
1364         struct dw_i3c_master *master = to_dw_i3c_master(m);
1365         int rc;
1366
1367         rc = i3c_master_disec_locked(m, dev->info.dyn_addr, I3C_CCC_EVENT_SIR);
1368         if (rc)
1369                 return rc;
1370
1371         dw_i3c_master_set_sir_enabled(master, dev, data->index, false);
1372
1373         pm_runtime_mark_last_busy(master->dev);
1374         pm_runtime_put_autosuspend(master->dev);
1375         return 0;
1376 }
1377
1378 static void dw_i3c_master_recycle_ibi_slot(struct i3c_dev_desc *dev,
1379                                            struct i3c_ibi_slot *slot)
1380 {
1381         struct dw_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev);
1382
1383         i3c_generic_ibi_recycle_slot(data->ibi_pool, slot);
1384 }
1385
1386 static void dw_i3c_master_drain_ibi_queue(struct dw_i3c_master *master,
1387                                           int len)
1388 {
1389         int i;
1390
1391         for (i = 0; i < DIV_ROUND_UP(len, 4); i++)
1392                 readl(master->regs + IBI_QUEUE_STATUS);
1393 }
1394
1395 static void dw_i3c_master_handle_ibi_sir(struct dw_i3c_master *master,
1396                                          u32 status)
1397 {
1398         struct dw_i3c_i2c_dev_data *data;
1399         struct i3c_ibi_slot *slot;
1400         struct i3c_dev_desc *dev;
1401         unsigned long flags;
1402         u8 addr, len;
1403         int idx;
1404
1405         addr = IBI_QUEUE_IBI_ADDR(status);
1406         len = IBI_QUEUE_STATUS_DATA_LEN(status);
1407
1408         /*
1409          * We be tempted to check the error status in bit 30; however, due
1410          * to the PEC errata workaround on some platform implementations (see
1411          * ast2600_i3c_set_dat_ibi()), those will almost always have a PEC
1412          * error on IBI payload data, as well as losing the last byte of
1413          * payload.
1414          *
1415          * If we implement error status checking on that bit, we may need
1416          * a new platform op to validate it.
1417          */
1418
1419         spin_lock_irqsave(&master->devs_lock, flags);
1420         idx = dw_i3c_master_get_addr_pos(master, addr);
1421         if (idx < 0) {
1422                 dev_dbg_ratelimited(&master->base.dev,
1423                          "IBI from unknown addr 0x%x\n", addr);
1424                 goto err_drain;
1425         }
1426
1427         dev = master->devs[idx].ibi_dev;
1428         if (!dev || !dev->ibi) {
1429                 dev_dbg_ratelimited(&master->base.dev,
1430                          "IBI from non-requested dev idx %d\n", idx);
1431                 goto err_drain;
1432         }
1433
1434         data = i3c_dev_get_master_data(dev);
1435         slot = i3c_generic_ibi_get_free_slot(data->ibi_pool);
1436         if (!slot) {
1437                 dev_dbg_ratelimited(&master->base.dev,
1438                                     "No IBI slots available\n");
1439                 goto err_drain;
1440         }
1441
1442         if (dev->ibi->max_payload_len < len) {
1443                 dev_dbg_ratelimited(&master->base.dev,
1444                                     "IBI payload len %d greater than max %d\n",
1445                                     len, dev->ibi->max_payload_len);
1446                 goto err_drain;
1447         }
1448
1449         if (len) {
1450                 dw_i3c_master_read_ibi_fifo(master, slot->data, len);
1451                 slot->len = len;
1452         }
1453         i3c_master_queue_ibi(dev, slot);
1454
1455         spin_unlock_irqrestore(&master->devs_lock, flags);
1456
1457         return;
1458
1459 err_drain:
1460         dw_i3c_master_drain_ibi_queue(master, len);
1461
1462         spin_unlock_irqrestore(&master->devs_lock, flags);
1463 }
1464
1465 /* "ibis": referring to In-Band Interrupts, and not
1466  * https://en.wikipedia.org/wiki/Australian_white_ibis. The latter should
1467  * not be handled.
1468  */
1469 static void dw_i3c_master_irq_handle_ibis(struct dw_i3c_master *master)
1470 {
1471         unsigned int i, len, n_ibis;
1472         u32 reg;
1473
1474         reg = readl(master->regs + QUEUE_STATUS_LEVEL);
1475         n_ibis = QUEUE_STATUS_IBI_STATUS_CNT(reg);
1476         if (!n_ibis)
1477                 return;
1478
1479         for (i = 0; i < n_ibis; i++) {
1480                 reg = readl(master->regs + IBI_QUEUE_STATUS);
1481
1482                 if (IBI_TYPE_SIRQ(reg)) {
1483                         dw_i3c_master_handle_ibi_sir(master, reg);
1484                 } else if (IBI_TYPE_HJ(reg)) {
1485                         queue_work(master->base.wq, &master->hj_work);
1486                 } else {
1487                         len = IBI_QUEUE_STATUS_DATA_LEN(reg);
1488                         dev_info(&master->base.dev,
1489                                  "unsupported IBI type 0x%lx len %d\n",
1490                                  IBI_QUEUE_STATUS_IBI_ID(reg), len);
1491                         dw_i3c_master_drain_ibi_queue(master, len);
1492                 }
1493         }
1494 }
1495
1496 static irqreturn_t dw_i3c_master_irq_handler(int irq, void *dev_id)
1497 {
1498         struct dw_i3c_master *master = dev_id;
1499         u32 status;
1500
1501         status = readl(master->regs + INTR_STATUS);
1502
1503         if (!(status & readl(master->regs + INTR_STATUS_EN))) {
1504                 writel(INTR_ALL, master->regs + INTR_STATUS);
1505                 return IRQ_NONE;
1506         }
1507
1508         spin_lock(&master->xferqueue.lock);
1509         dw_i3c_master_end_xfer_locked(master, status);
1510         if (status & INTR_TRANSFER_ERR_STAT)
1511                 writel(INTR_TRANSFER_ERR_STAT, master->regs + INTR_STATUS);
1512         spin_unlock(&master->xferqueue.lock);
1513
1514         if (status & INTR_IBI_THLD_STAT)
1515                 dw_i3c_master_irq_handle_ibis(master);
1516
1517         return IRQ_HANDLED;
1518 }
1519
1520 static const struct i3c_master_controller_ops dw_mipi_i3c_ops = {
1521         .bus_init = dw_i3c_master_bus_init,
1522         .bus_cleanup = dw_i3c_master_bus_cleanup,
1523         .attach_i3c_dev = dw_i3c_master_attach_i3c_dev,
1524         .reattach_i3c_dev = dw_i3c_master_reattach_i3c_dev,
1525         .detach_i3c_dev = dw_i3c_master_detach_i3c_dev,
1526         .do_daa = dw_i3c_master_daa,
1527         .supports_ccc_cmd = dw_i3c_master_supports_ccc_cmd,
1528         .send_ccc_cmd = dw_i3c_master_send_ccc_cmd,
1529         .priv_xfers = dw_i3c_master_priv_xfers,
1530         .attach_i2c_dev = dw_i3c_master_attach_i2c_dev,
1531         .detach_i2c_dev = dw_i3c_master_detach_i2c_dev,
1532         .i2c_xfers = dw_i3c_master_i2c_xfers,
1533         .request_ibi = dw_i3c_master_request_ibi,
1534         .free_ibi = dw_i3c_master_free_ibi,
1535         .enable_ibi = dw_i3c_master_enable_ibi,
1536         .disable_ibi = dw_i3c_master_disable_ibi,
1537         .recycle_ibi_slot = dw_i3c_master_recycle_ibi_slot,
1538         .enable_hotjoin = dw_i3c_master_enable_hotjoin,
1539         .disable_hotjoin = dw_i3c_master_disable_hotjoin,
1540 };
1541
1542 /* default platform ops implementations */
1543 static int dw_i3c_platform_init_nop(struct dw_i3c_master *i3c)
1544 {
1545         return 0;
1546 }
1547
1548 static void dw_i3c_platform_set_dat_ibi_nop(struct dw_i3c_master *i3c,
1549                                         struct i3c_dev_desc *dev,
1550                                         bool enable, u32 *dat)
1551 {
1552 }
1553
1554 static const struct dw_i3c_platform_ops dw_i3c_platform_ops_default = {
1555         .init = dw_i3c_platform_init_nop,
1556         .set_dat_ibi = dw_i3c_platform_set_dat_ibi_nop,
1557 };
1558
1559 static void dw_i3c_hj_work(struct work_struct *work)
1560 {
1561         struct dw_i3c_master *master =
1562                 container_of(work, typeof(*master), hj_work);
1563
1564         i3c_master_do_daa(&master->base);
1565 }
1566
1567 int dw_i3c_common_probe(struct dw_i3c_master *master,
1568                         struct platform_device *pdev)
1569 {
1570         int ret, irq;
1571
1572         if (!master->platform_ops)
1573                 master->platform_ops = &dw_i3c_platform_ops_default;
1574
1575         master->dev = &pdev->dev;
1576
1577         master->regs = devm_platform_ioremap_resource(pdev, 0);
1578         if (IS_ERR(master->regs))
1579                 return PTR_ERR(master->regs);
1580
1581         master->core_clk = devm_clk_get_enabled(&pdev->dev, NULL);
1582         if (IS_ERR(master->core_clk))
1583                 return PTR_ERR(master->core_clk);
1584
1585         master->pclk = devm_clk_get_optional_enabled(&pdev->dev, "pclk");
1586         if (IS_ERR(master->pclk))
1587                 return PTR_ERR(master->pclk);
1588
1589         master->core_rst = devm_reset_control_get_optional_exclusive(&pdev->dev,
1590                                                                     "core_rst");
1591         if (IS_ERR(master->core_rst))
1592                 return PTR_ERR(master->core_rst);
1593
1594         reset_control_deassert(master->core_rst);
1595
1596         spin_lock_init(&master->xferqueue.lock);
1597         INIT_LIST_HEAD(&master->xferqueue.list);
1598
1599         writel(INTR_ALL, master->regs + INTR_STATUS);
1600         irq = platform_get_irq(pdev, 0);
1601         ret = devm_request_irq(&pdev->dev, irq,
1602                                dw_i3c_master_irq_handler, 0,
1603                                dev_name(&pdev->dev), master);
1604         if (ret)
1605                 goto err_assert_rst;
1606
1607         platform_set_drvdata(pdev, master);
1608
1609         pm_runtime_set_autosuspend_delay(&pdev->dev, RPM_AUTOSUSPEND_TIMEOUT);
1610         pm_runtime_use_autosuspend(&pdev->dev);
1611         pm_runtime_set_active(&pdev->dev);
1612         pm_runtime_enable(&pdev->dev);
1613
1614         /* Information regarding the FIFOs/QUEUEs depth */
1615         ret = readl(master->regs + QUEUE_STATUS_LEVEL);
1616         master->caps.cmdfifodepth = QUEUE_STATUS_LEVEL_CMD(ret);
1617
1618         ret = readl(master->regs + DATA_BUFFER_STATUS_LEVEL);
1619         master->caps.datafifodepth = DATA_BUFFER_STATUS_LEVEL_TX(ret);
1620
1621         ret = readl(master->regs + DEVICE_ADDR_TABLE_POINTER);
1622         master->datstartaddr = ret;
1623         master->maxdevs = ret >> 16;
1624         master->free_pos = GENMASK(master->maxdevs - 1, 0);
1625
1626         master->quirks = (unsigned long)device_get_match_data(&pdev->dev);
1627
1628         INIT_WORK(&master->hj_work, dw_i3c_hj_work);
1629         ret = i3c_master_register(&master->base, &pdev->dev,
1630                                   &dw_mipi_i3c_ops, false);
1631         if (ret)
1632                 goto err_disable_pm;
1633
1634         return 0;
1635
1636 err_disable_pm:
1637         pm_runtime_disable(&pdev->dev);
1638         pm_runtime_set_suspended(&pdev->dev);
1639         pm_runtime_dont_use_autosuspend(&pdev->dev);
1640
1641 err_assert_rst:
1642         reset_control_assert(master->core_rst);
1643
1644         return ret;
1645 }
1646 EXPORT_SYMBOL_GPL(dw_i3c_common_probe);
1647
1648 void dw_i3c_common_remove(struct dw_i3c_master *master)
1649 {
1650         i3c_master_unregister(&master->base);
1651
1652         pm_runtime_disable(master->dev);
1653         pm_runtime_set_suspended(master->dev);
1654         pm_runtime_dont_use_autosuspend(master->dev);
1655 }
1656 EXPORT_SYMBOL_GPL(dw_i3c_common_remove);
1657
1658 /* base platform implementation */
1659
1660 static int dw_i3c_probe(struct platform_device *pdev)
1661 {
1662         struct dw_i3c_master *master;
1663
1664         master = devm_kzalloc(&pdev->dev, sizeof(*master), GFP_KERNEL);
1665         if (!master)
1666                 return -ENOMEM;
1667
1668         return dw_i3c_common_probe(master, pdev);
1669 }
1670
1671 static void dw_i3c_remove(struct platform_device *pdev)
1672 {
1673         struct dw_i3c_master *master = platform_get_drvdata(pdev);
1674
1675         dw_i3c_common_remove(master);
1676 }
1677
1678 static void dw_i3c_master_restore_addrs(struct dw_i3c_master *master)
1679 {
1680         u32 pos, reg_val;
1681
1682         writel(DEV_ADDR_DYNAMIC_ADDR_VALID | DEV_ADDR_DYNAMIC(master->dev_addr),
1683                master->regs + DEVICE_ADDR);
1684
1685         for (pos = 0; pos < master->maxdevs; pos++) {
1686                 if (master->free_pos & BIT(pos))
1687                         continue;
1688
1689                 if (master->devs[pos].is_i2c_addr)
1690                         reg_val = DEV_ADDR_TABLE_LEGACY_I2C_DEV |
1691                                DEV_ADDR_TABLE_STATIC_ADDR(master->devs[pos].addr);
1692                 else
1693                         reg_val = DEV_ADDR_TABLE_DYNAMIC_ADDR(master->devs[pos].addr);
1694
1695                 writel(reg_val, master->regs + DEV_ADDR_TABLE_LOC(master->datstartaddr, pos));
1696         }
1697 }
1698
1699 static void dw_i3c_master_restore_timing_regs(struct dw_i3c_master *master)
1700 {
1701         /* AMD platform specific OD and PP timings */
1702         if (master->quirks & AMD_I3C_OD_PP_TIMING)
1703                 amd_configure_od_pp_quirk(master);
1704
1705         writel(master->i3c_pp_timing, master->regs + SCL_I3C_PP_TIMING);
1706         writel(master->bus_free_timing, master->regs + BUS_FREE_TIMING);
1707         writel(master->i3c_od_timing, master->regs + SCL_I3C_OD_TIMING);
1708         writel(master->ext_lcnt_timing, master->regs + SCL_EXT_LCNT_TIMING);
1709
1710         if (master->i2c_slv_prsnt) {
1711                 writel(master->i2c_fmp_timing, master->regs + SCL_I2C_FMP_TIMING);
1712                 writel(master->i2c_fm_timing, master->regs + SCL_I2C_FM_TIMING);
1713         }
1714 }
1715
1716 static int dw_i3c_master_enable_clks(struct dw_i3c_master *master)
1717 {
1718         int ret = 0;
1719
1720         ret = clk_prepare_enable(master->core_clk);
1721         if (ret)
1722                 return ret;
1723
1724         ret = clk_prepare_enable(master->pclk);
1725         if (ret) {
1726                 clk_disable_unprepare(master->core_clk);
1727                 return ret;
1728         }
1729
1730         return 0;
1731 }
1732
1733 static inline void dw_i3c_master_disable_clks(struct dw_i3c_master *master)
1734 {
1735         clk_disable_unprepare(master->pclk);
1736         clk_disable_unprepare(master->core_clk);
1737 }
1738
1739 static int __maybe_unused dw_i3c_master_runtime_suspend(struct device *dev)
1740 {
1741         struct dw_i3c_master *master = dev_get_drvdata(dev);
1742
1743         dw_i3c_master_disable(master);
1744
1745         reset_control_assert(master->core_rst);
1746         dw_i3c_master_disable_clks(master);
1747         pinctrl_pm_select_sleep_state(dev);
1748         return 0;
1749 }
1750
1751 static int __maybe_unused dw_i3c_master_runtime_resume(struct device *dev)
1752 {
1753         struct dw_i3c_master *master = dev_get_drvdata(dev);
1754
1755         pinctrl_pm_select_default_state(dev);
1756         dw_i3c_master_enable_clks(master);
1757         reset_control_deassert(master->core_rst);
1758
1759         dw_i3c_master_set_intr_regs(master);
1760         dw_i3c_master_restore_timing_regs(master);
1761         dw_i3c_master_restore_addrs(master);
1762
1763         dw_i3c_master_enable(master);
1764         return 0;
1765 }
1766
1767 static const struct dev_pm_ops dw_i3c_pm_ops = {
1768         SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume)
1769         SET_RUNTIME_PM_OPS(dw_i3c_master_runtime_suspend, dw_i3c_master_runtime_resume, NULL)
1770 };
1771
1772 static const struct of_device_id dw_i3c_master_of_match[] = {
1773         { .compatible = "snps,dw-i3c-master-1.00a", },
1774         {},
1775 };
1776 MODULE_DEVICE_TABLE(of, dw_i3c_master_of_match);
1777
1778 static const struct acpi_device_id amd_i3c_device_match[] = {
1779         { "AMDI0015", AMD_I3C_OD_PP_TIMING },
1780         { }
1781 };
1782 MODULE_DEVICE_TABLE(acpi, amd_i3c_device_match);
1783
1784 static struct platform_driver dw_i3c_driver = {
1785         .probe = dw_i3c_probe,
1786         .remove = dw_i3c_remove,
1787         .driver = {
1788                 .name = "dw-i3c-master",
1789                 .of_match_table = dw_i3c_master_of_match,
1790                 .acpi_match_table = amd_i3c_device_match,
1791                 .pm = &dw_i3c_pm_ops,
1792         },
1793 };
1794 module_platform_driver(dw_i3c_driver);
1795
1796 MODULE_AUTHOR("Vitor Soares <[email protected]>");
1797 MODULE_DESCRIPTION("DesignWare MIPI I3C driver");
1798 MODULE_LICENSE("GPL v2");