drivers/i3c/master/svc-i3c-master.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Silvaco dual-role I3C master driver
   4  *
   5  * Copyright (C) 2020 Silvaco
   6  * Author: Miquel RAYNAL <[email protected]>
   7  * Based on a work from: Conor Culhane <[email protected]>
   8  */
   9
  10 #include <linux/bitfield.h>
  11 #include <linux/clk.h>
  12 #include <linux/completion.h>
  13 #include <linux/errno.h>
  14 #include <linux/i3c/master.h>
  15 #include <linux/interrupt.h>
  16 #include <linux/iopoll.h>
  17 #include <linux/list.h>
  18 #include <linux/module.h>
  19 #include <linux/of.h>
  20 #include <linux/pinctrl/consumer.h>
  21 #include <linux/platform_device.h>
  22 #include <linux/pm_runtime.h>
  23
  24 /* Master Mode Registers */
  25 #define SVC_I3C_MCONFIG      0x000
  26 #define   SVC_I3C_MCONFIG_MASTER_EN BIT(0)
  27 #define   SVC_I3C_MCONFIG_DISTO(x) FIELD_PREP(BIT(3), (x))
  28 #define   SVC_I3C_MCONFIG_HKEEP(x) FIELD_PREP(GENMASK(5, 4), (x))
  29 #define   SVC_I3C_MCONFIG_ODSTOP(x) FIELD_PREP(BIT(6), (x))
  30 #define   SVC_I3C_MCONFIG_PPBAUD(x) FIELD_PREP(GENMASK(11, 8), (x))
  31 #define   SVC_I3C_MCONFIG_PPLOW(x) FIELD_PREP(GENMASK(15, 12), (x))
  32 #define   SVC_I3C_MCONFIG_ODBAUD(x) FIELD_PREP(GENMASK(23, 16), (x))
  33 #define   SVC_I3C_MCONFIG_ODHPP(x) FIELD_PREP(BIT(24), (x))
  34 #define   SVC_I3C_MCONFIG_SKEW(x) FIELD_PREP(GENMASK(27, 25), (x))
  35 #define   SVC_I3C_MCONFIG_I2CBAUD(x) FIELD_PREP(GENMASK(31, 28), (x))
  36
  37 #define SVC_I3C_MCTRL        0x084
  38 #define   SVC_I3C_MCTRL_REQUEST_MASK GENMASK(2, 0)
  39 #define   SVC_I3C_MCTRL_REQUEST_NONE 0
  40 #define   SVC_I3C_MCTRL_REQUEST_START_ADDR 1
  41 #define   SVC_I3C_MCTRL_REQUEST_STOP 2
  42 #define   SVC_I3C_MCTRL_REQUEST_IBI_ACKNACK 3
  43 #define   SVC_I3C_MCTRL_REQUEST_PROC_DAA 4
  44 #define   SVC_I3C_MCTRL_REQUEST_AUTO_IBI 7
  45 #define   SVC_I3C_MCTRL_TYPE_I3C 0
  46 #define   SVC_I3C_MCTRL_TYPE_I2C BIT(4)
  47 #define   SVC_I3C_MCTRL_IBIRESP_AUTO 0
  48 #define   SVC_I3C_MCTRL_IBIRESP_ACK_WITHOUT_BYTE 0
  49 #define   SVC_I3C_MCTRL_IBIRESP_ACK_WITH_BYTE BIT(7)
  50 #define   SVC_I3C_MCTRL_IBIRESP_NACK BIT(6)
  51 #define   SVC_I3C_MCTRL_IBIRESP_MANUAL GENMASK(7, 6)
  52 #define   SVC_I3C_MCTRL_DIR(x) FIELD_PREP(BIT(8), (x))
  53 #define   SVC_I3C_MCTRL_DIR_WRITE 0
  54 #define   SVC_I3C_MCTRL_DIR_READ 1
  55 #define   SVC_I3C_MCTRL_ADDR(x) FIELD_PREP(GENMASK(15, 9), (x))
  56 #define   SVC_I3C_MCTRL_RDTERM(x) FIELD_PREP(GENMASK(23, 16), (x))
  57
  58 #define SVC_I3C_MSTATUS      0x088
  59 #define   SVC_I3C_MSTATUS_STATE(x) FIELD_GET(GENMASK(2, 0), (x))
  60 #define   SVC_I3C_MSTATUS_STATE_DAA(x) (SVC_I3C_MSTATUS_STATE(x) == 5)
  61 #define   SVC_I3C_MSTATUS_STATE_IDLE(x) (SVC_I3C_MSTATUS_STATE(x) == 0)
  62 #define   SVC_I3C_MSTATUS_BETWEEN(x) FIELD_GET(BIT(4), (x))
  63 #define   SVC_I3C_MSTATUS_NACKED(x) FIELD_GET(BIT(5), (x))
  64 #define   SVC_I3C_MSTATUS_IBITYPE(x) FIELD_GET(GENMASK(7, 6), (x))
  65 #define   SVC_I3C_MSTATUS_IBITYPE_IBI 1
  66 #define   SVC_I3C_MSTATUS_IBITYPE_MASTER_REQUEST 2
  67 #define   SVC_I3C_MSTATUS_IBITYPE_HOT_JOIN 3
  68 #define   SVC_I3C_MINT_SLVSTART BIT(8)
  69 #define   SVC_I3C_MINT_MCTRLDONE BIT(9)
  70 #define   SVC_I3C_MINT_COMPLETE BIT(10)
  71 #define   SVC_I3C_MINT_RXPEND BIT(11)
  72 #define   SVC_I3C_MINT_TXNOTFULL BIT(12)
  73 #define   SVC_I3C_MINT_IBIWON BIT(13)
  74 #define   SVC_I3C_MINT_ERRWARN BIT(15)
  75 #define   SVC_I3C_MSTATUS_SLVSTART(x) FIELD_GET(SVC_I3C_MINT_SLVSTART, (x))
  76 #define   SVC_I3C_MSTATUS_MCTRLDONE(x) FIELD_GET(SVC_I3C_MINT_MCTRLDONE, (x))
  77 #define   SVC_I3C_MSTATUS_COMPLETE(x) FIELD_GET(SVC_I3C_MINT_COMPLETE, (x))
  78 #define   SVC_I3C_MSTATUS_RXPEND(x) FIELD_GET(SVC_I3C_MINT_RXPEND, (x))
  79 #define   SVC_I3C_MSTATUS_TXNOTFULL(x) FIELD_GET(SVC_I3C_MINT_TXNOTFULL, (x))
  80 #define   SVC_I3C_MSTATUS_IBIWON(x) FIELD_GET(SVC_I3C_MINT_IBIWON, (x))
  81 #define   SVC_I3C_MSTATUS_ERRWARN(x) FIELD_GET(SVC_I3C_MINT_ERRWARN, (x))
  82 #define   SVC_I3C_MSTATUS_IBIADDR(x) FIELD_GET(GENMASK(30, 24), (x))
  83
  84 #define SVC_I3C_IBIRULES     0x08C
  85 #define   SVC_I3C_IBIRULES_ADDR(slot, addr) FIELD_PREP(GENMASK(29, 0), \
  86                                                        ((addr) & 0x3F) << ((slot) * 6))
  87 #define   SVC_I3C_IBIRULES_ADDRS 5
  88 #define   SVC_I3C_IBIRULES_MSB0 BIT(30)
  89 #define   SVC_I3C_IBIRULES_NOBYTE BIT(31)
  90 #define   SVC_I3C_IBIRULES_MANDBYTE 0
  91 #define SVC_I3C_MINTSET      0x090
  92 #define SVC_I3C_MINTCLR      0x094
  93 #define SVC_I3C_MINTMASKED   0x098
  94 #define SVC_I3C_MERRWARN     0x09C
  95 #define   SVC_I3C_MERRWARN_NACK BIT(2)
  96 #define   SVC_I3C_MERRWARN_TIMEOUT BIT(20)
  97 #define SVC_I3C_MDMACTRL     0x0A0
  98 #define SVC_I3C_MDATACTRL    0x0AC
  99 #define   SVC_I3C_MDATACTRL_FLUSHTB BIT(0)
 100 #define   SVC_I3C_MDATACTRL_FLUSHRB BIT(1)
 101 #define   SVC_I3C_MDATACTRL_UNLOCK_TRIG BIT(3)
 102 #define   SVC_I3C_MDATACTRL_TXTRIG_FIFO_NOT_FULL GENMASK(5, 4)
 103 #define   SVC_I3C_MDATACTRL_RXTRIG_FIFO_NOT_EMPTY 0
 104 #define   SVC_I3C_MDATACTRL_RXCOUNT(x) FIELD_GET(GENMASK(28, 24), (x))
 105 #define   SVC_I3C_MDATACTRL_TXFULL BIT(30)
 106 #define   SVC_I3C_MDATACTRL_RXEMPTY BIT(31)
 107
 108 #define SVC_I3C_MWDATAB      0x0B0
 109 #define   SVC_I3C_MWDATAB_END BIT(8)
 110
 111 #define SVC_I3C_MWDATABE     0x0B4
 112 #define SVC_I3C_MWDATAH      0x0B8
 113 #define SVC_I3C_MWDATAHE     0x0BC
 114 #define SVC_I3C_MRDATAB      0x0C0
 115 #define SVC_I3C_MRDATAH      0x0C8
 116 #define SVC_I3C_MWMSG_SDR    0x0D0
 117 #define SVC_I3C_MRMSG_SDR    0x0D4
 118 #define SVC_I3C_MWMSG_DDR    0x0D8
 119 #define SVC_I3C_MRMSG_DDR    0x0DC
 120
 121 #define SVC_I3C_MDYNADDR     0x0E4
 122 #define   SVC_MDYNADDR_VALID BIT(0)
 123 #define   SVC_MDYNADDR_ADDR(x) FIELD_PREP(GENMASK(7, 1), (x))
 124
 125 #define SVC_I3C_MAX_DEVS 32
 126 #define SVC_I3C_PM_TIMEOUT_MS 1000
 127
 128 /* This parameter depends on the implementation and may be tuned */
 129 #define SVC_I3C_FIFO_SIZE 16
 130 #define SVC_I3C_PPBAUD_MAX 15
 131 #define SVC_I3C_QUICK_I2C_CLK 4170000
 132
 133 #define SVC_I3C_EVENT_IBI       GENMASK(7, 0)
 134 #define SVC_I3C_EVENT_HOTJOIN   BIT(31)
 135
 136 struct svc_i3c_cmd {
 137         u8 addr;
 138         bool rnw;
 139         u8 *in;
 140         const void *out;
 141         unsigned int len;
 142         unsigned int actual_len;
 143         struct i3c_priv_xfer *xfer;
 144         bool continued;
 145 };
 146
 147 struct svc_i3c_xfer {
 148         struct list_head node;
 149         struct completion comp;
 150         int ret;
 151         unsigned int type;
 152         unsigned int ncmds;
 153         struct svc_i3c_cmd cmds[] __counted_by(ncmds);
 154 };
 155
 156 struct svc_i3c_regs_save {
 157         u32 mconfig;
 158         u32 mdynaddr;
 159 };
 160
 161 /**
 162  * struct svc_i3c_master - Silvaco I3C Master structure
 163  * @base: I3C master controller
 164  * @dev: Corresponding device
 165  * @regs: Memory mapping
 166  * @saved_regs: Volatile values for PM operations
 167  * @free_slots: Bit array of available slots
 168  * @addrs: Array containing the dynamic addresses of each attached device
 169  * @descs: Array of descriptors, one per attached device
 170  * @hj_work: Hot-join work
 171  * @ibi_work: IBI work
 172  * @irq: Main interrupt
 173  * @pclk: System clock
 174  * @fclk: Fast clock (bus)
 175  * @sclk: Slow clock (other events)
 176  * @xferqueue: Transfer queue structure
 177  * @xferqueue.list: List member
 178  * @xferqueue.cur: Current ongoing transfer
 179  * @xferqueue.lock: Queue lock
 180  * @ibi: IBI structure
 181  * @ibi.num_slots: Number of slots available in @ibi.slots
 182  * @ibi.slots: Available IBI slots
 183  * @ibi.tbq_slot: To be queued IBI slot
 184  * @ibi.lock: IBI lock
 185  * @lock: Transfer lock, protect between IBI work thread and callbacks from master
 186  * @enabled_events: Bit masks for enable events (IBI, HotJoin).
 187  * @mctrl_config: Configuration value in SVC_I3C_MCTRL for setting speed back.
 188  */
 189 struct svc_i3c_master {
 190         struct i3c_master_controller base;
 191         struct device *dev;
 192         void __iomem *regs;
 193         struct svc_i3c_regs_save saved_regs;
 194         u32 free_slots;
 195         u8 addrs[SVC_I3C_MAX_DEVS];
 196         struct i3c_dev_desc *descs[SVC_I3C_MAX_DEVS];
 197         struct work_struct hj_work;
 198         struct work_struct ibi_work;
 199         int irq;
 200         struct clk *pclk;
 201         struct clk *fclk;
 202         struct clk *sclk;
 203         struct {
 204                 struct list_head list;
 205                 struct svc_i3c_xfer *cur;
 206                 /* Prevent races between transfers */
 207                 spinlock_t lock;
 208         } xferqueue;
 209         struct {
 210                 unsigned int num_slots;
 211                 struct i3c_dev_desc **slots;
 212                 struct i3c_ibi_slot *tbq_slot;
 213                 /* Prevent races within IBI handlers */
 214                 spinlock_t lock;
 215         } ibi;
 216         struct mutex lock;
 217         u32 enabled_events;
 218         u32 mctrl_config;
 219 };
 220
 221 /**
 222  * struct svc_i3c_i2c_dev_data - Device specific data
 223  * @index: Index in the master tables corresponding to this device
 224  * @ibi: IBI slot index in the master structure
 225  * @ibi_pool: IBI pool associated to this device
 226  */
 227 struct svc_i3c_i2c_dev_data {
 228         u8 index;
 229         int ibi;
 230         struct i3c_generic_ibi_pool *ibi_pool;
 231 };
 232
 233 static inline bool is_events_enabled(struct svc_i3c_master *master, u32 mask)
 234 {
 235         return !!(master->enabled_events & mask);
 236 }
 237
 238 static bool svc_i3c_master_error(struct svc_i3c_master *master)
 239 {
 240         u32 mstatus, merrwarn;
 241
 242         mstatus = readl(master->regs + SVC_I3C_MSTATUS);
 243         if (SVC_I3C_MSTATUS_ERRWARN(mstatus)) {
 244                 merrwarn = readl(master->regs + SVC_I3C_MERRWARN);
 245                 writel(merrwarn, master->regs + SVC_I3C_MERRWARN);
 246
 247                 /* Ignore timeout error */
 248                 if (merrwarn & SVC_I3C_MERRWARN_TIMEOUT) {
 249                         dev_dbg(master->dev, "Warning condition: MSTATUS 0x%08x, MERRWARN 0x%08x\n",
 250                                 mstatus, merrwarn);
 251                         return false;
 252                 }
 253
 254                 dev_err(master->dev,
 255                         "Error condition: MSTATUS 0x%08x, MERRWARN 0x%08x\n",
 256                         mstatus, merrwarn);
 257
 258                 return true;
 259         }
 260
 261         return false;
 262 }
 263
 264 static void svc_i3c_master_enable_interrupts(struct svc_i3c_master *master, u32 mask)
 265 {
 266         writel(mask, master->regs + SVC_I3C_MINTSET);
 267 }
 268
 269 static void svc_i3c_master_disable_interrupts(struct svc_i3c_master *master)
 270 {
 271         u32 mask = readl(master->regs + SVC_I3C_MINTSET);
 272
 273         writel(mask, master->regs + SVC_I3C_MINTCLR);
 274 }
 275
 276 static void svc_i3c_master_clear_merrwarn(struct svc_i3c_master *master)
 277 {
 278         /* Clear pending warnings */
 279         writel(readl(master->regs + SVC_I3C_MERRWARN),
 280                master->regs + SVC_I3C_MERRWARN);
 281 }
 282
 283 static void svc_i3c_master_flush_fifo(struct svc_i3c_master *master)
 284 {
 285         /* Flush FIFOs */
 286         writel(SVC_I3C_MDATACTRL_FLUSHTB | SVC_I3C_MDATACTRL_FLUSHRB,
 287                master->regs + SVC_I3C_MDATACTRL);
 288 }
 289
 290 static void svc_i3c_master_reset_fifo_trigger(struct svc_i3c_master *master)
 291 {
 292         u32 reg;
 293
 294         /* Set RX and TX tigger levels, flush FIFOs */
 295         reg = SVC_I3C_MDATACTRL_FLUSHTB |
 296               SVC_I3C_MDATACTRL_FLUSHRB |
 297               SVC_I3C_MDATACTRL_UNLOCK_TRIG |
 298               SVC_I3C_MDATACTRL_TXTRIG_FIFO_NOT_FULL |
 299               SVC_I3C_MDATACTRL_RXTRIG_FIFO_NOT_EMPTY;
 300         writel(reg, master->regs + SVC_I3C_MDATACTRL);
 301 }
 302
 303 static void svc_i3c_master_reset(struct svc_i3c_master *master)
 304 {
 305         svc_i3c_master_clear_merrwarn(master);
 306         svc_i3c_master_reset_fifo_trigger(master);
 307         svc_i3c_master_disable_interrupts(master);
 308 }
 309
 310 static inline struct svc_i3c_master *
 311 to_svc_i3c_master(struct i3c_master_controller *master)
 312 {
 313         return container_of(master, struct svc_i3c_master, base);
 314 }
 315
 316 static void svc_i3c_master_hj_work(struct work_struct *work)
 317 {
 318         struct svc_i3c_master *master;
 319
 320         master = container_of(work, struct svc_i3c_master, hj_work);
 321         i3c_master_do_daa(&master->base);
 322 }
 323
 324 static struct i3c_dev_desc *
 325 svc_i3c_master_dev_from_addr(struct svc_i3c_master *master,
 326                              unsigned int ibiaddr)
 327 {
 328         int i;
 329
 330         for (i = 0; i < SVC_I3C_MAX_DEVS; i++)
 331                 if (master->addrs[i] == ibiaddr)
 332                         break;
 333
 334         if (i == SVC_I3C_MAX_DEVS)
 335                 return NULL;
 336
 337         return master->descs[i];
 338 }
 339
 340 static void svc_i3c_master_emit_stop(struct svc_i3c_master *master)
 341 {
 342         writel(SVC_I3C_MCTRL_REQUEST_STOP, master->regs + SVC_I3C_MCTRL);
 343
 344         /*
 345          * This delay is necessary after the emission of a stop, otherwise eg.
 346          * repeating IBIs do not get detected. There is a note in the manual
 347          * about it, stating that the stop condition might not be settled
 348          * correctly if a start condition follows too rapidly.
 349          */
 350         udelay(1);
 351 }
 352
 353 static int svc_i3c_master_handle_ibi(struct svc_i3c_master *master,
 354                                      struct i3c_dev_desc *dev)
 355 {
 356         struct svc_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev);
 357         struct i3c_ibi_slot *slot;
 358         unsigned int count;
 359         u32 mdatactrl;
 360         int ret, val;
 361         u8 *buf;
 362
 363         slot = i3c_generic_ibi_get_free_slot(data->ibi_pool);
 364         if (!slot)
 365                 return -ENOSPC;
 366
 367         slot->len = 0;
 368         buf = slot->data;
 369
 370         ret = readl_relaxed_poll_timeout(master->regs + SVC_I3C_MSTATUS, val,
 371                                                 SVC_I3C_MSTATUS_COMPLETE(val), 0, 1000);
 372         if (ret) {
 373                 dev_err(master->dev, "Timeout when polling for COMPLETE\n");
 374                 return ret;
 375         }
 376
 377         while (SVC_I3C_MSTATUS_RXPEND(readl(master->regs + SVC_I3C_MSTATUS))  &&
 378                slot->len < SVC_I3C_FIFO_SIZE) {
 379                 mdatactrl = readl(master->regs + SVC_I3C_MDATACTRL);
 380                 count = SVC_I3C_MDATACTRL_RXCOUNT(mdatactrl);
 381                 readsl(master->regs + SVC_I3C_MRDATAB, buf, count);
 382                 slot->len += count;
 383                 buf += count;
 384         }
 385
 386         master->ibi.tbq_slot = slot;
 387
 388         return 0;
 389 }
 390
 391 static int svc_i3c_master_ack_ibi(struct svc_i3c_master *master,
 392                                    bool mandatory_byte)
 393 {
 394         unsigned int ibi_ack_nack;
 395         u32 reg;
 396
 397         ibi_ack_nack = SVC_I3C_MCTRL_REQUEST_IBI_ACKNACK;
 398         if (mandatory_byte)
 399                 ibi_ack_nack |= SVC_I3C_MCTRL_IBIRESP_ACK_WITH_BYTE;
 400         else
 401                 ibi_ack_nack |= SVC_I3C_MCTRL_IBIRESP_ACK_WITHOUT_BYTE;
 402
 403         writel(ibi_ack_nack, master->regs + SVC_I3C_MCTRL);
 404
 405         return readl_poll_timeout_atomic(master->regs + SVC_I3C_MSTATUS, reg,
 406                                          SVC_I3C_MSTATUS_MCTRLDONE(reg), 1, 1000);
 407
 408 }
 409
 410 static int svc_i3c_master_nack_ibi(struct svc_i3c_master *master)
 411 {
 412         int ret;
 413         u32 reg;
 414
 415         writel(SVC_I3C_MCTRL_REQUEST_IBI_ACKNACK |
 416                SVC_I3C_MCTRL_IBIRESP_NACK,
 417                master->regs + SVC_I3C_MCTRL);
 418
 419         ret = readl_poll_timeout_atomic(master->regs + SVC_I3C_MSTATUS, reg,
 420                                         SVC_I3C_MSTATUS_MCTRLDONE(reg), 1, 1000);
 421         return ret;
 422 }
 423
 424 static int svc_i3c_master_handle_ibi_won(struct svc_i3c_master *master, u32 mstatus)
 425 {
 426         u32 ibitype;
 427         int ret = 0;
 428
 429         ibitype = SVC_I3C_MSTATUS_IBITYPE(mstatus);
 430
 431         writel(SVC_I3C_MINT_IBIWON, master->regs + SVC_I3C_MSTATUS);
 432
 433         /* Hardware can't auto emit NACK for hot join and master request */
 434         switch (ibitype) {
 435         case SVC_I3C_MSTATUS_IBITYPE_HOT_JOIN:
 436         case SVC_I3C_MSTATUS_IBITYPE_MASTER_REQUEST:
 437                 ret = svc_i3c_master_nack_ibi(master);
 438         }
 439
 440         return ret;
 441 }
 442
 443 static void svc_i3c_master_ibi_work(struct work_struct *work)
 444 {
 445         struct svc_i3c_master *master = container_of(work, struct svc_i3c_master, ibi_work);
 446         struct svc_i3c_i2c_dev_data *data;
 447         unsigned int ibitype, ibiaddr;
 448         struct i3c_dev_desc *dev;
 449         u32 status, val;
 450         int ret;
 451
 452         /*
 453          * According to I3C spec ver 1.1, 09-Jun-2021, section 5.1.2.5:
 454          *
 455          * The I3C Controller shall hold SCL low while the Bus is in ACK/NACK Phase of I3C/I2C
 456          * transfer. But maximum stall time is 100us. The IRQs have to be disabled to prevent
 457          * schedule during the whole I3C transaction, otherwise, the I3C bus timeout may happen if
 458          * any irq or schedule happen during transaction.
 459          */
 460         guard(spinlock_irqsave)(&master->xferqueue.lock);
 461
 462         /*
 463          * IBIWON may be set before SVC_I3C_MCTRL_REQUEST_AUTO_IBI, causing
 464          * readl_relaxed_poll_timeout() to return immediately. Consequently,
 465          * ibitype will be 0 since it was last updated only after the 8th SCL
 466          * cycle, leading to missed client IBI handlers.
 467          *
 468          * A typical scenario is when IBIWON occurs and bus arbitration is lost
 469          * at svc_i3c_master_priv_xfers().
 470          *
 471          * Clear SVC_I3C_MINT_IBIWON before sending SVC_I3C_MCTRL_REQUEST_AUTO_IBI.
 472          */
 473         writel(SVC_I3C_MINT_IBIWON, master->regs + SVC_I3C_MSTATUS);
 474
 475         /* Acknowledge the incoming interrupt with the AUTOIBI mechanism */
 476         writel(SVC_I3C_MCTRL_REQUEST_AUTO_IBI |
 477                SVC_I3C_MCTRL_IBIRESP_AUTO,
 478                master->regs + SVC_I3C_MCTRL);
 479
 480         /* Wait for IBIWON, should take approximately 100us */
 481         ret = readl_relaxed_poll_timeout_atomic(master->regs + SVC_I3C_MSTATUS, val,
 482                                          SVC_I3C_MSTATUS_IBIWON(val), 0, 100);
 483         if (ret) {
 484                 dev_err(master->dev, "Timeout when polling for IBIWON\n");
 485                 svc_i3c_master_emit_stop(master);
 486                 goto reenable_ibis;
 487         }
 488
 489         status = readl(master->regs + SVC_I3C_MSTATUS);
 490         ibitype = SVC_I3C_MSTATUS_IBITYPE(status);
 491         ibiaddr = SVC_I3C_MSTATUS_IBIADDR(status);
 492
 493         /* Handle the critical responses to IBI's */
 494         switch (ibitype) {
 495         case SVC_I3C_MSTATUS_IBITYPE_IBI:
 496                 dev = svc_i3c_master_dev_from_addr(master, ibiaddr);
 497                 if (!dev || !is_events_enabled(master, SVC_I3C_EVENT_IBI))
 498                         svc_i3c_master_nack_ibi(master);
 499                 else
 500                         svc_i3c_master_handle_ibi(master, dev);
 501                 break;
 502         case SVC_I3C_MSTATUS_IBITYPE_HOT_JOIN:
 503                 if (is_events_enabled(master, SVC_I3C_EVENT_HOTJOIN))
 504                         svc_i3c_master_ack_ibi(master, false);
 505                 else
 506                         svc_i3c_master_nack_ibi(master);
 507                 break;
 508         case SVC_I3C_MSTATUS_IBITYPE_MASTER_REQUEST:
 509                 svc_i3c_master_nack_ibi(master);
 510                 break;
 511         default:
 512                 break;
 513         }
 514
 515         /*
 516          * If an error happened, we probably got interrupted and the exchange
 517          * timedout. In this case we just drop everything, emit a stop and wait
 518          * for the slave to interrupt again.
 519          */
 520         if (svc_i3c_master_error(master)) {
 521                 if (master->ibi.tbq_slot) {
 522                         data = i3c_dev_get_master_data(dev);
 523                         i3c_generic_ibi_recycle_slot(data->ibi_pool,
 524                                                      master->ibi.tbq_slot);
 525                         master->ibi.tbq_slot = NULL;
 526                 }
 527
 528                 svc_i3c_master_emit_stop(master);
 529
 530                 goto reenable_ibis;
 531         }
 532
 533         /* Handle the non critical tasks */
 534         switch (ibitype) {
 535         case SVC_I3C_MSTATUS_IBITYPE_IBI:
 536                 if (dev) {
 537                         i3c_master_queue_ibi(dev, master->ibi.tbq_slot);
 538                         master->ibi.tbq_slot = NULL;
 539                 }
 540                 svc_i3c_master_emit_stop(master);
 541                 break;
 542         case SVC_I3C_MSTATUS_IBITYPE_HOT_JOIN:
 543                 svc_i3c_master_emit_stop(master);
 544                 if (is_events_enabled(master, SVC_I3C_EVENT_HOTJOIN))
 545                         queue_work(master->base.wq, &master->hj_work);
 546                 break;
 547         case SVC_I3C_MSTATUS_IBITYPE_MASTER_REQUEST:
 548         default:
 549                 break;
 550         }
 551
 552 reenable_ibis:
 553         svc_i3c_master_enable_interrupts(master, SVC_I3C_MINT_SLVSTART);
 554 }
 555
 556 static irqreturn_t svc_i3c_master_irq_handler(int irq, void *dev_id)
 557 {
 558         struct svc_i3c_master *master = (struct svc_i3c_master *)dev_id;
 559         u32 active = readl(master->regs + SVC_I3C_MSTATUS);
 560
 561         if (!SVC_I3C_MSTATUS_SLVSTART(active))
 562                 return IRQ_NONE;
 563
 564         /* Clear the interrupt status */
 565         writel(SVC_I3C_MINT_SLVSTART, master->regs + SVC_I3C_MSTATUS);
 566
 567         svc_i3c_master_disable_interrupts(master);
 568
 569         /* Handle the interrupt in a non atomic context */
 570         queue_work(master->base.wq, &master->ibi_work);
 571
 572         return IRQ_HANDLED;
 573 }
 574
 575 static int svc_i3c_master_set_speed(struct i3c_master_controller *m,
 576                                      enum i3c_open_drain_speed speed)
 577 {
 578         struct svc_i3c_master *master = to_svc_i3c_master(m);
 579         struct i3c_bus *bus = i3c_master_get_bus(&master->base);
 580         u32 ppbaud, odbaud, odhpp, mconfig;
 581         unsigned long fclk_rate;
 582         int ret;
 583
 584         ret = pm_runtime_resume_and_get(master->dev);
 585         if (ret < 0) {
 586                 dev_err(master->dev, "<%s> Cannot get runtime PM.\n", __func__);
 587                 return ret;
 588         }
 589
 590         switch (speed) {
 591         case I3C_OPEN_DRAIN_SLOW_SPEED:
 592                 fclk_rate = clk_get_rate(master->fclk);
 593                 if (!fclk_rate) {
 594                         ret = -EINVAL;
 595                         goto rpm_out;
 596                 }
 597                 /*
 598                  * Set 50% duty-cycle I2C speed to I3C OPEN-DRAIN mode, so the first
 599                  * broadcast address is visible to all I2C/I3C devices on the I3C bus.
 600                  * I3C device working as a I2C device will turn off its 50ns Spike
 601                  * Filter to change to I3C mode.
 602                  */
 603                 mconfig = master->mctrl_config;
 604                 ppbaud = FIELD_GET(GENMASK(11, 8), mconfig);
 605                 odhpp = 0;
 606                 odbaud = DIV_ROUND_UP(fclk_rate, bus->scl_rate.i2c * (2 + 2 * ppbaud)) - 1;
 607                 mconfig &= ~GENMASK(24, 16);
 608                 mconfig |= SVC_I3C_MCONFIG_ODBAUD(odbaud) | SVC_I3C_MCONFIG_ODHPP(odhpp);
 609                 writel(mconfig, master->regs + SVC_I3C_MCONFIG);
 610                 break;
 611         case I3C_OPEN_DRAIN_NORMAL_SPEED:
 612                 writel(master->mctrl_config, master->regs + SVC_I3C_MCONFIG);
 613                 break;
 614         }
 615
 616 rpm_out:
 617         pm_runtime_mark_last_busy(master->dev);
 618         pm_runtime_put_autosuspend(master->dev);
 619
 620         return ret;
 621 }
 622
 623 static int svc_i3c_master_bus_init(struct i3c_master_controller *m)
 624 {
 625         struct svc_i3c_master *master = to_svc_i3c_master(m);
 626         struct i3c_bus *bus = i3c_master_get_bus(m);
 627         struct i3c_device_info info = {};
 628         unsigned long fclk_rate, fclk_period_ns;
 629         unsigned long i2c_period_ns, i2c_scl_rate, i3c_scl_rate;
 630         unsigned int high_period_ns, od_low_period_ns;
 631         u32 ppbaud, pplow, odhpp, odbaud, odstop, i2cbaud, reg;
 632         int ret;
 633
 634         ret = pm_runtime_resume_and_get(master->dev);
 635         if (ret < 0) {
 636                 dev_err(master->dev,
 637                         "<%s> cannot resume i3c bus master, err: %d\n",
 638                         __func__, ret);
 639                 return ret;
 640         }
 641
 642         /* Timings derivation */
 643         fclk_rate = clk_get_rate(master->fclk);
 644         if (!fclk_rate) {
 645                 ret = -EINVAL;
 646                 goto rpm_out;
 647         }
 648
 649         fclk_period_ns = DIV_ROUND_UP(1000000000, fclk_rate);
 650         i2c_period_ns = DIV_ROUND_UP(1000000000, bus->scl_rate.i2c);
 651         i2c_scl_rate = bus->scl_rate.i2c;
 652         i3c_scl_rate = bus->scl_rate.i3c;
 653
 654         /*
 655          * Using I3C Push-Pull mode, target is 12.5MHz/80ns period.
 656          * Simplest configuration is using a 50% duty-cycle of 40ns.
 657          */
 658         ppbaud = DIV_ROUND_UP(fclk_rate / 2, i3c_scl_rate) - 1;
 659         pplow = 0;
 660
 661         /*
 662          * Using I3C Open-Drain mode, target is 4.17MHz/240ns with a
 663          * duty-cycle tuned so that high levels are filetered out by
 664          * the 50ns filter (target being 40ns).
 665          */
 666         odhpp = 1;
 667         high_period_ns = (ppbaud + 1) * fclk_period_ns;
 668         odbaud = DIV_ROUND_UP(fclk_rate, SVC_I3C_QUICK_I2C_CLK * (1 + ppbaud)) - 2;
 669         od_low_period_ns = (odbaud + 1) * high_period_ns;
 670
 671         switch (bus->mode) {
 672         case I3C_BUS_MODE_PURE:
 673                 i2cbaud = 0;
 674                 odstop = 0;
 675                 break;
 676         case I3C_BUS_MODE_MIXED_FAST:
 677                 /*
 678                  * Using I2C Fm+ mode, target is 1MHz/1000ns, the difference
 679                  * between the high and low period does not really matter.
 680                  */
 681                 i2cbaud = DIV_ROUND_UP(i2c_period_ns, od_low_period_ns) - 2;
 682                 odstop = 1;
 683                 break;
 684         case I3C_BUS_MODE_MIXED_LIMITED:
 685         case I3C_BUS_MODE_MIXED_SLOW:
 686                 /* I3C PP + I3C OP + I2C OP both use i2c clk rate */
 687                 if (ppbaud > SVC_I3C_PPBAUD_MAX) {
 688                         ppbaud = SVC_I3C_PPBAUD_MAX;
 689                         pplow =  DIV_ROUND_UP(fclk_rate, i3c_scl_rate) - (2 + 2 * ppbaud);
 690                 }
 691
 692                 high_period_ns = (ppbaud + 1) * fclk_period_ns;
 693                 odhpp = 0;
 694                 odbaud = DIV_ROUND_UP(fclk_rate, i2c_scl_rate * (2 + 2 * ppbaud)) - 1;
 695
 696                 od_low_period_ns = (odbaud + 1) * high_period_ns;
 697                 i2cbaud = DIV_ROUND_UP(i2c_period_ns, od_low_period_ns) - 2;
 698                 odstop = 1;
 699                 break;
 700         default:
 701                 goto rpm_out;
 702         }
 703
 704         reg = SVC_I3C_MCONFIG_MASTER_EN |
 705               SVC_I3C_MCONFIG_DISTO(0) |
 706               SVC_I3C_MCONFIG_HKEEP(0) |
 707               SVC_I3C_MCONFIG_ODSTOP(odstop) |
 708               SVC_I3C_MCONFIG_PPBAUD(ppbaud) |
 709               SVC_I3C_MCONFIG_PPLOW(pplow) |
 710               SVC_I3C_MCONFIG_ODBAUD(odbaud) |
 711               SVC_I3C_MCONFIG_ODHPP(odhpp) |
 712               SVC_I3C_MCONFIG_SKEW(0) |
 713               SVC_I3C_MCONFIG_I2CBAUD(i2cbaud);
 714         writel(reg, master->regs + SVC_I3C_MCONFIG);
 715
 716         master->mctrl_config = reg;
 717         /* Master core's registration */
 718         ret = i3c_master_get_free_addr(m, 0);
 719         if (ret < 0)
 720                 goto rpm_out;
 721
 722         info.dyn_addr = ret;
 723
 724         writel(SVC_MDYNADDR_VALID | SVC_MDYNADDR_ADDR(info.dyn_addr),
 725                master->regs + SVC_I3C_MDYNADDR);
 726
 727         ret = i3c_master_set_info(&master->base, &info);
 728         if (ret)
 729                 goto rpm_out;
 730
 731 rpm_out:
 732         pm_runtime_mark_last_busy(master->dev);
 733         pm_runtime_put_autosuspend(master->dev);
 734
 735         return ret;
 736 }
 737
 738 static void svc_i3c_master_bus_cleanup(struct i3c_master_controller *m)
 739 {
 740         struct svc_i3c_master *master = to_svc_i3c_master(m);
 741         int ret;
 742
 743         ret = pm_runtime_resume_and_get(master->dev);
 744         if (ret < 0) {
 745                 dev_err(master->dev, "<%s> Cannot get runtime PM.\n", __func__);
 746                 return;
 747         }
 748
 749         svc_i3c_master_disable_interrupts(master);
 750
 751         /* Disable master */
 752         writel(0, master->regs + SVC_I3C_MCONFIG);
 753
 754         pm_runtime_mark_last_busy(master->dev);
 755         pm_runtime_put_autosuspend(master->dev);
 756 }
 757
 758 static int svc_i3c_master_reserve_slot(struct svc_i3c_master *master)
 759 {
 760         unsigned int slot;
 761
 762         if (!(master->free_slots & GENMASK(SVC_I3C_MAX_DEVS - 1, 0)))
 763                 return -ENOSPC;
 764
 765         slot = ffs(master->free_slots) - 1;
 766
 767         master->free_slots &= ~BIT(slot);
 768
 769         return slot;
 770 }
 771
 772 static void svc_i3c_master_release_slot(struct svc_i3c_master *master,
 773                                         unsigned int slot)
 774 {
 775         master->free_slots |= BIT(slot);
 776 }
 777
 778 static int svc_i3c_master_attach_i3c_dev(struct i3c_dev_desc *dev)
 779 {
 780         struct i3c_master_controller *m = i3c_dev_get_master(dev);
 781         struct svc_i3c_master *master = to_svc_i3c_master(m);
 782         struct svc_i3c_i2c_dev_data *data;
 783         int slot;
 784
 785         slot = svc_i3c_master_reserve_slot(master);
 786         if (slot < 0)
 787                 return slot;
 788
 789         data = kzalloc(sizeof(*data), GFP_KERNEL);
 790         if (!data) {
 791                 svc_i3c_master_release_slot(master, slot);
 792                 return -ENOMEM;
 793         }
 794
 795         data->ibi = -1;
 796         data->index = slot;
 797         master->addrs[slot] = dev->info.dyn_addr ? dev->info.dyn_addr :
 798                                                    dev->info.static_addr;
 799         master->descs[slot] = dev;
 800
 801         i3c_dev_set_master_data(dev, data);
 802
 803         return 0;
 804 }
 805
 806 static int svc_i3c_master_reattach_i3c_dev(struct i3c_dev_desc *dev,
 807                                            u8 old_dyn_addr)
 808 {
 809         struct i3c_master_controller *m = i3c_dev_get_master(dev);
 810         struct svc_i3c_master *master = to_svc_i3c_master(m);
 811         struct svc_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev);
 812
 813         master->addrs[data->index] = dev->info.dyn_addr ? dev->info.dyn_addr :
 814                                                           dev->info.static_addr;
 815
 816         return 0;
 817 }
 818
 819 static void svc_i3c_master_detach_i3c_dev(struct i3c_dev_desc *dev)
 820 {
 821         struct svc_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev);
 822         struct i3c_master_controller *m = i3c_dev_get_master(dev);
 823         struct svc_i3c_master *master = to_svc_i3c_master(m);
 824
 825         master->addrs[data->index] = 0;
 826         svc_i3c_master_release_slot(master, data->index);
 827
 828         kfree(data);
 829 }
 830
 831 static int svc_i3c_master_attach_i2c_dev(struct i2c_dev_desc *dev)
 832 {
 833         struct i3c_master_controller *m = i2c_dev_get_master(dev);
 834         struct svc_i3c_master *master = to_svc_i3c_master(m);
 835         struct svc_i3c_i2c_dev_data *data;
 836         int slot;
 837
 838         slot = svc_i3c_master_reserve_slot(master);
 839         if (slot < 0)
 840                 return slot;
 841
 842         data = kzalloc(sizeof(*data), GFP_KERNEL);
 843         if (!data) {
 844                 svc_i3c_master_release_slot(master, slot);
 845                 return -ENOMEM;
 846         }
 847
 848         data->index = slot;
 849         master->addrs[slot] = dev->addr;
 850
 851         i2c_dev_set_master_data(dev, data);
 852
 853         return 0;
 854 }
 855
 856 static void svc_i3c_master_detach_i2c_dev(struct i2c_dev_desc *dev)
 857 {
 858         struct svc_i3c_i2c_dev_data *data = i2c_dev_get_master_data(dev);
 859         struct i3c_master_controller *m = i2c_dev_get_master(dev);
 860         struct svc_i3c_master *master = to_svc_i3c_master(m);
 861
 862         svc_i3c_master_release_slot(master, data->index);
 863
 864         kfree(data);
 865 }
 866
 867 static int svc_i3c_master_readb(struct svc_i3c_master *master, u8 *dst,
 868                                 unsigned int len)
 869 {
 870         int ret, i;
 871         u32 reg;
 872
 873         for (i = 0; i < len; i++) {
 874                 ret = readl_poll_timeout_atomic(master->regs + SVC_I3C_MSTATUS,
 875                                                 reg,
 876                                                 SVC_I3C_MSTATUS_RXPEND(reg),
 877                                                 0, 1000);
 878                 if (ret)
 879                         return ret;
 880
 881                 dst[i] = readl(master->regs + SVC_I3C_MRDATAB);
 882         }
 883
 884         return 0;
 885 }
 886
 887 static int svc_i3c_master_do_daa_locked(struct svc_i3c_master *master,
 888                                         u8 *addrs, unsigned int *count)
 889 {
 890         u64 prov_id[SVC_I3C_MAX_DEVS] = {}, nacking_prov_id = 0;
 891         unsigned int dev_nb = 0, last_addr = 0;
 892         u32 reg;
 893         int ret, i;
 894
 895         while (true) {
 896                 /* clean SVC_I3C_MINT_IBIWON w1c bits */
 897                 writel(SVC_I3C_MINT_IBIWON, master->regs + SVC_I3C_MSTATUS);
 898
 899                 /* SVC_I3C_MCTRL_REQUEST_PROC_DAA have two mode, ENTER DAA or PROCESS DAA.
 900                  *
 901                  * ENTER DAA:
 902                  *   1 will issue START, 7E, ENTDAA, and then emits 7E/R to process first target.
 903                  *   2 Stops just before the new Dynamic Address (DA) is to be emitted.
 904                  *
 905                  * PROCESS DAA:
 906                  *   1 The DA is written using MWDATAB or ADDR bits 6:0.
 907                  *   2 ProcessDAA is requested again to write the new address, and then starts the
 908                  *     next (START, 7E, ENTDAA)  unless marked to STOP; an MSTATUS indicating NACK
 909                  *     means DA was not accepted (e.g. parity error). If PROCESSDAA is NACKed on the
 910                  *     7E/R, which means no more Slaves need a DA, then a COMPLETE will be signaled
 911                  *     (along with DONE), and a STOP issued automatically.
 912                  */
 913                 writel(SVC_I3C_MCTRL_REQUEST_PROC_DAA |
 914                        SVC_I3C_MCTRL_TYPE_I3C |
 915                        SVC_I3C_MCTRL_IBIRESP_NACK |
 916                        SVC_I3C_MCTRL_DIR(SVC_I3C_MCTRL_DIR_WRITE),
 917                        master->regs + SVC_I3C_MCTRL);
 918
 919                 /*
 920                  * Either one slave will send its ID, or the assignment process
 921                  * is done.
 922                  */
 923                 ret = readl_poll_timeout_atomic(master->regs + SVC_I3C_MSTATUS,
 924                                                 reg,
 925                                                 SVC_I3C_MSTATUS_RXPEND(reg) |
 926                                                 SVC_I3C_MSTATUS_MCTRLDONE(reg),
 927                                                 1, 1000);
 928                 if (ret)
 929                         break;
 930
 931                 if (SVC_I3C_MSTATUS_RXPEND(reg)) {
 932                         u8 data[6];
 933
 934                         /*
 935                          * We only care about the 48-bit provisioned ID yet to
 936                          * be sure a device does not nack an address twice.
 937                          * Otherwise, we would just need to flush the RX FIFO.
 938                          */
 939                         ret = svc_i3c_master_readb(master, data, 6);
 940                         if (ret)
 941                                 break;
 942
 943                         for (i = 0; i < 6; i++)
 944                                 prov_id[dev_nb] |= (u64)(data[i]) << (8 * (5 - i));
 945
 946                         /* We do not care about the BCR and DCR yet */
 947                         ret = svc_i3c_master_readb(master, data, 2);
 948                         if (ret)
 949                                 break;
 950                 } else if (SVC_I3C_MSTATUS_IBIWON(reg)) {
 951                         ret = svc_i3c_master_handle_ibi_won(master, reg);
 952                         if (ret)
 953                                 break;
 954                         continue;
 955                 } else if (SVC_I3C_MSTATUS_MCTRLDONE(reg)) {
 956                         if (SVC_I3C_MSTATUS_STATE_IDLE(reg) &&
 957                             SVC_I3C_MSTATUS_COMPLETE(reg)) {
 958                                 /*
 959                                  * All devices received and acked they dynamic
 960                                  * address, this is the natural end of the DAA
 961                                  * procedure.
 962                                  *
 963                                  * Hardware will auto emit STOP at this case.
 964                                  */
 965                                 *count = dev_nb;
 966                                 return 0;
 967
 968                         } else if (SVC_I3C_MSTATUS_NACKED(reg)) {
 969                                 /* No I3C devices attached */
 970                                 if (dev_nb == 0) {
 971                                         /*
 972                                          * Hardware can't treat first NACK for ENTAA as normal
 973                                          * COMPLETE. So need manual emit STOP.
 974                                          */
 975                                         ret = 0;
 976                                         *count = 0;
 977                                         break;
 978                                 }
 979
 980                                 /*
 981                                  * A slave device nacked the address, this is
 982                                  * allowed only once, DAA will be stopped and
 983                                  * then resumed. The same device is supposed to
 984                                  * answer again immediately and shall ack the
 985                                  * address this time.
 986                                  */
 987                                 if (prov_id[dev_nb] == nacking_prov_id) {
 988                                         ret = -EIO;
 989                                         break;
 990                                 }
 991
 992                                 dev_nb--;
 993                                 nacking_prov_id = prov_id[dev_nb];
 994                                 svc_i3c_master_emit_stop(master);
 995
 996                                 continue;
 997                         } else {
 998                                 break;
 999                         }
1000                 }
1001
1002                 /* Wait for the slave to be ready to receive its address */
1003                 ret = readl_poll_timeout_atomic(master->regs + SVC_I3C_MSTATUS,
1004                                                 reg,
1005                                                 SVC_I3C_MSTATUS_MCTRLDONE(reg) &&
1006                                                 SVC_I3C_MSTATUS_STATE_DAA(reg) &&
1007                                                 SVC_I3C_MSTATUS_BETWEEN(reg),
1008                                                 0, 1000);
1009                 if (ret)
1010                         break;
1011
1012                 /* Give the slave device a suitable dynamic address */
1013                 ret = i3c_master_get_free_addr(&master->base, last_addr + 1);
1014                 if (ret < 0)
1015                         break;
1016
1017                 addrs[dev_nb] = ret;
1018                 dev_dbg(master->dev, "DAA: device %d assigned to 0x%02x\n",
1019                         dev_nb, addrs[dev_nb]);
1020
1021                 writel(addrs[dev_nb], master->regs + SVC_I3C_MWDATAB);
1022                 last_addr = addrs[dev_nb++];
1023         }
1024
1025         /* Need manual issue STOP except for Complete condition */
1026         svc_i3c_master_emit_stop(master);
1027         return ret;
1028 }
1029
1030 static int svc_i3c_update_ibirules(struct svc_i3c_master *master)
1031 {
1032         struct i3c_dev_desc *dev;
1033         u32 reg_mbyte = 0, reg_nobyte = SVC_I3C_IBIRULES_NOBYTE;
1034         unsigned int mbyte_addr_ok = 0, mbyte_addr_ko = 0, nobyte_addr_ok = 0,
1035                 nobyte_addr_ko = 0;
1036         bool list_mbyte = false, list_nobyte = false;
1037
1038         /* Create the IBIRULES register for both cases */
1039         i3c_bus_for_each_i3cdev(&master->base.bus, dev) {
1040                 if (I3C_BCR_DEVICE_ROLE(dev->info.bcr) == I3C_BCR_I3C_MASTER)
1041                         continue;
1042
1043                 if (dev->info.bcr & I3C_BCR_IBI_PAYLOAD) {
1044                         reg_mbyte |= SVC_I3C_IBIRULES_ADDR(mbyte_addr_ok,
1045                                                            dev->info.dyn_addr);
1046
1047                         /* IBI rules cannot be applied to devices with MSb=1 */
1048                         if (dev->info.dyn_addr & BIT(7))
1049                                 mbyte_addr_ko++;
1050                         else
1051                                 mbyte_addr_ok++;
1052                 } else {
1053                         reg_nobyte |= SVC_I3C_IBIRULES_ADDR(nobyte_addr_ok,
1054                                                             dev->info.dyn_addr);
1055
1056                         /* IBI rules cannot be applied to devices with MSb=1 */
1057                         if (dev->info.dyn_addr & BIT(7))
1058                                 nobyte_addr_ko++;
1059                         else
1060                                 nobyte_addr_ok++;
1061                 }
1062         }
1063
1064         /* Device list cannot be handled by hardware */
1065         if (!mbyte_addr_ko && mbyte_addr_ok <= SVC_I3C_IBIRULES_ADDRS)
1066                 list_mbyte = true;
1067
1068         if (!nobyte_addr_ko && nobyte_addr_ok <= SVC_I3C_IBIRULES_ADDRS)
1069                 list_nobyte = true;
1070
1071         /* No list can be properly handled, return an error */
1072         if (!list_mbyte && !list_nobyte)
1073                 return -ERANGE;
1074
1075         /* Pick the first list that can be handled by hardware, randomly */
1076         if (list_mbyte)
1077                 writel(reg_mbyte, master->regs + SVC_I3C_IBIRULES);
1078         else
1079                 writel(reg_nobyte, master->regs + SVC_I3C_IBIRULES);
1080
1081         return 0;
1082 }
1083
1084 static int svc_i3c_master_do_daa(struct i3c_master_controller *m)
1085 {
1086         struct svc_i3c_master *master = to_svc_i3c_master(m);
1087         u8 addrs[SVC_I3C_MAX_DEVS];
1088         unsigned long flags;
1089         unsigned int dev_nb;
1090         int ret, i;
1091
1092         ret = pm_runtime_resume_and_get(master->dev);
1093         if (ret < 0) {
1094                 dev_err(master->dev, "<%s> Cannot get runtime PM.\n", __func__);
1095                 return ret;
1096         }
1097
1098         spin_lock_irqsave(&master->xferqueue.lock, flags);
1099         ret = svc_i3c_master_do_daa_locked(master, addrs, &dev_nb);
1100         spin_unlock_irqrestore(&master->xferqueue.lock, flags);
1101
1102         svc_i3c_master_clear_merrwarn(master);
1103         if (ret)
1104                 goto rpm_out;
1105
1106         /*
1107          * Register all devices who participated to the core
1108          *
1109          * If two devices (A and B) are detected in DAA and address 0xa is assigned to
1110          * device A and 0xb to device B, a failure in i3c_master_add_i3c_dev_locked()
1111          * for device A (addr: 0xa) could prevent device B (addr: 0xb) from being
1112          * registered on the bus. The I3C stack might still consider 0xb a free
1113          * address. If a subsequent Hotjoin occurs, 0xb might be assigned to Device A,
1114          * causing both devices A and B to use the same address 0xb, violating the I3C
1115          * specification.
1116          *
1117          * The return value for i3c_master_add_i3c_dev_locked() should not be checked
1118          * because subsequent steps will scan the entire I3C bus, independent of
1119          * whether i3c_master_add_i3c_dev_locked() returns success.
1120          *
1121          * If device A registration fails, there is still a chance to register device
1122          * B. i3c_master_add_i3c_dev_locked() can reset DAA if a failure occurs while
1123          * retrieving device information.
1124          */
1125         for (i = 0; i < dev_nb; i++)
1126                 i3c_master_add_i3c_dev_locked(m, addrs[i]);
1127
1128         /* Configure IBI auto-rules */
1129         ret = svc_i3c_update_ibirules(master);
1130         if (ret)
1131                 dev_err(master->dev, "Cannot handle such a list of devices");
1132
1133 rpm_out:
1134         pm_runtime_mark_last_busy(master->dev);
1135         pm_runtime_put_autosuspend(master->dev);
1136
1137         return ret;
1138 }
1139
1140 static int svc_i3c_master_read(struct svc_i3c_master *master,
1141                                u8 *in, unsigned int len)
1142 {
1143         int offset = 0, i;
1144         u32 mdctrl, mstatus;
1145         bool completed = false;
1146         unsigned int count;
1147         unsigned long start = jiffies;
1148
1149         while (!completed) {
1150                 mstatus = readl(master->regs + SVC_I3C_MSTATUS);
1151                 if (SVC_I3C_MSTATUS_COMPLETE(mstatus) != 0)
1152                         completed = true;
1153
1154                 if (time_after(jiffies, start + msecs_to_jiffies(1000))) {
1155                         dev_dbg(master->dev, "I3C read timeout\n");
1156                         return -ETIMEDOUT;
1157                 }
1158
1159                 mdctrl = readl(master->regs + SVC_I3C_MDATACTRL);
1160                 count = SVC_I3C_MDATACTRL_RXCOUNT(mdctrl);
1161                 if (offset + count > len) {
1162                         dev_err(master->dev, "I3C receive length too long!\n");
1163                         return -EINVAL;
1164                 }
1165                 for (i = 0; i < count; i++)
1166                         in[offset + i] = readl(master->regs + SVC_I3C_MRDATAB);
1167
1168                 offset += count;
1169         }
1170
1171         return offset;
1172 }
1173
1174 static int svc_i3c_master_write(struct svc_i3c_master *master,
1175                                 const u8 *out, unsigned int len)
1176 {
1177         int offset = 0, ret;
1178         u32 mdctrl;
1179
1180         while (offset < len) {
1181                 ret = readl_poll_timeout(master->regs + SVC_I3C_MDATACTRL,
1182                                          mdctrl,
1183                                          !(mdctrl & SVC_I3C_MDATACTRL_TXFULL),
1184                                          0, 1000);
1185                 if (ret)
1186                         return ret;
1187
1188                 /*
1189                  * The last byte to be sent over the bus must either have the
1190                  * "end" bit set or be written in MWDATABE.
1191                  */
1192                 if (likely(offset < (len - 1)))
1193                         writel(out[offset++], master->regs + SVC_I3C_MWDATAB);
1194                 else
1195                         writel(out[offset++], master->regs + SVC_I3C_MWDATABE);
1196         }
1197
1198         return 0;
1199 }
1200
1201 static int svc_i3c_master_xfer(struct svc_i3c_master *master,
1202                                bool rnw, unsigned int xfer_type, u8 addr,
1203                                u8 *in, const u8 *out, unsigned int xfer_len,
1204                                unsigned int *actual_len, bool continued)
1205 {
1206         int retry = 2;
1207         u32 reg;
1208         int ret;
1209
1210         /* clean SVC_I3C_MINT_IBIWON w1c bits */
1211         writel(SVC_I3C_MINT_IBIWON, master->regs + SVC_I3C_MSTATUS);
1212
1213
1214         while (retry--) {
1215                 writel(SVC_I3C_MCTRL_REQUEST_START_ADDR |
1216                        xfer_type |
1217                        SVC_I3C_MCTRL_IBIRESP_NACK |
1218                        SVC_I3C_MCTRL_DIR(rnw) |
1219                        SVC_I3C_MCTRL_ADDR(addr) |
1220                        SVC_I3C_MCTRL_RDTERM(*actual_len),
1221                        master->regs + SVC_I3C_MCTRL);
1222
1223                 ret = readl_poll_timeout(master->regs + SVC_I3C_MSTATUS, reg,
1224                                  SVC_I3C_MSTATUS_MCTRLDONE(reg), 0, 1000);
1225                 if (ret)
1226                         goto emit_stop;
1227
1228                 /*
1229                  * According to I3C spec ver 1.1.1, 5.1.2.2.3 Consequence of Controller Starting a
1230                  * Frame with I3C Target Address.
1231                  *
1232                  * The I3C Controller normally should start a Frame, the Address may be arbitrated,
1233                  * and so the Controller shall monitor to see whether an In-Band Interrupt request,
1234                  * a Controller Role Request (i.e., Secondary Controller requests to become the
1235                  * Active Controller), or a Hot-Join Request has been made.
1236                  *
1237                  * If missed IBIWON check, the wrong data will be return. When IBIWON happen, issue
1238                  * repeat start. Address arbitrate only happen at START, never happen at REPEAT
1239                  * start.
1240                  */
1241                 if (SVC_I3C_MSTATUS_IBIWON(reg)) {
1242                         ret = svc_i3c_master_handle_ibi_won(master, reg);
1243                         if (ret)
1244                                 goto emit_stop;
1245                         continue;
1246                 }
1247
1248                 if (readl(master->regs + SVC_I3C_MERRWARN) & SVC_I3C_MERRWARN_NACK) {
1249                         /*
1250                          * According to I3C Spec 1.1.1, 11-Jun-2021, section: 5.1.2.2.3.
1251                          * If the Controller chooses to start an I3C Message with an I3C Dynamic
1252                          * Address, then special provisions shall be made because that same I3C
1253                          * Target may be initiating an IBI or a Controller Role Request. So, one of
1254                          * three things may happen: (skip 1, 2)
1255                          *
1256                          * 3. The Addresses match and the RnW bits also match, and so neither
1257                          * Controller nor Target will ACK since both are expecting the other side to
1258                          * provide ACK. As a result, each side might think it had "won" arbitration,
1259                          * but neither side would continue, as each would subsequently see that the
1260                          * other did not provide ACK.
1261                          * ...
1262                          * For either value of RnW: Due to the NACK, the Controller shall defer the
1263                          * Private Write or Private Read, and should typically transmit the Target
1264                          * Address again after a Repeated START (i.e., the next one or any one prior
1265                          * to a STOP in the Frame). Since the Address Header following a Repeated
1266                          * START is not arbitrated, the Controller will always win (see Section
1267                          * 5.1.2.2.4).
1268                          */
1269                         if (retry && addr != 0x7e) {
1270                                 writel(SVC_I3C_MERRWARN_NACK, master->regs + SVC_I3C_MERRWARN);
1271                         } else {
1272                                 ret = -ENXIO;
1273                                 *actual_len = 0;
1274                                 goto emit_stop;
1275                         }
1276                 } else {
1277                         break;
1278                 }
1279         }
1280
1281         if (rnw)
1282                 ret = svc_i3c_master_read(master, in, xfer_len);
1283         else
1284                 ret = svc_i3c_master_write(master, out, xfer_len);
1285         if (ret < 0)
1286                 goto emit_stop;
1287
1288         if (rnw)
1289                 *actual_len = ret;
1290
1291         ret = readl_poll_timeout(master->regs + SVC_I3C_MSTATUS, reg,
1292                                  SVC_I3C_MSTATUS_COMPLETE(reg), 0, 1000);
1293         if (ret)
1294                 goto emit_stop;
1295
1296         writel(SVC_I3C_MINT_COMPLETE, master->regs + SVC_I3C_MSTATUS);
1297
1298         if (!continued) {
1299                 svc_i3c_master_emit_stop(master);
1300
1301                 /* Wait idle if stop is sent. */
1302                 readl_poll_timeout(master->regs + SVC_I3C_MSTATUS, reg,
1303                                    SVC_I3C_MSTATUS_STATE_IDLE(reg), 0, 1000);
1304         }
1305
1306         return 0;
1307
1308 emit_stop:
1309         svc_i3c_master_emit_stop(master);
1310         svc_i3c_master_clear_merrwarn(master);
1311
1312         return ret;
1313 }
1314
1315 static struct svc_i3c_xfer *
1316 svc_i3c_master_alloc_xfer(struct svc_i3c_master *master, unsigned int ncmds)
1317 {
1318         struct svc_i3c_xfer *xfer;
1319
1320         xfer = kzalloc(struct_size(xfer, cmds, ncmds), GFP_KERNEL);
1321         if (!xfer)
1322                 return NULL;
1323
1324         INIT_LIST_HEAD(&xfer->node);
1325         xfer->ncmds = ncmds;
1326         xfer->ret = -ETIMEDOUT;
1327
1328         return xfer;
1329 }
1330
1331 static void svc_i3c_master_free_xfer(struct svc_i3c_xfer *xfer)
1332 {
1333         kfree(xfer);
1334 }
1335
1336 static void svc_i3c_master_dequeue_xfer_locked(struct svc_i3c_master *master,
1337                                                struct svc_i3c_xfer *xfer)
1338 {
1339         if (master->xferqueue.cur == xfer)
1340                 master->xferqueue.cur = NULL;
1341         else
1342                 list_del_init(&xfer->node);
1343 }
1344
1345 static void svc_i3c_master_dequeue_xfer(struct svc_i3c_master *master,
1346                                         struct svc_i3c_xfer *xfer)
1347 {
1348         unsigned long flags;
1349
1350         spin_lock_irqsave(&master->xferqueue.lock, flags);
1351         svc_i3c_master_dequeue_xfer_locked(master, xfer);
1352         spin_unlock_irqrestore(&master->xferqueue.lock, flags);
1353 }
1354
1355 static void svc_i3c_master_start_xfer_locked(struct svc_i3c_master *master)
1356 {
1357         struct svc_i3c_xfer *xfer = master->xferqueue.cur;
1358         int ret, i;
1359
1360         if (!xfer)
1361                 return;
1362
1363         svc_i3c_master_clear_merrwarn(master);
1364         svc_i3c_master_flush_fifo(master);
1365
1366         for (i = 0; i < xfer->ncmds; i++) {
1367                 struct svc_i3c_cmd *cmd = &xfer->cmds[i];
1368
1369                 ret = svc_i3c_master_xfer(master, cmd->rnw, xfer->type,
1370                                           cmd->addr, cmd->in, cmd->out,
1371                                           cmd->len, &cmd->actual_len,
1372                                           cmd->continued);
1373                 /* cmd->xfer is NULL if I2C or CCC transfer */
1374                 if (cmd->xfer)
1375                         cmd->xfer->actual_len = cmd->actual_len;
1376
1377                 if (ret)
1378                         break;
1379         }
1380
1381         xfer->ret = ret;
1382         complete(&xfer->comp);
1383
1384         if (ret < 0)
1385                 svc_i3c_master_dequeue_xfer_locked(master, xfer);
1386
1387         xfer = list_first_entry_or_null(&master->xferqueue.list,
1388                                         struct svc_i3c_xfer,
1389                                         node);
1390         if (xfer)
1391                 list_del_init(&xfer->node);
1392
1393         master->xferqueue.cur = xfer;
1394         svc_i3c_master_start_xfer_locked(master);
1395 }
1396
1397 static void svc_i3c_master_enqueue_xfer(struct svc_i3c_master *master,
1398                                         struct svc_i3c_xfer *xfer)
1399 {
1400         unsigned long flags;
1401         int ret;
1402
1403         ret = pm_runtime_resume_and_get(master->dev);
1404         if (ret < 0) {
1405                 dev_err(master->dev, "<%s> Cannot get runtime PM.\n", __func__);
1406                 return;
1407         }
1408
1409         init_completion(&xfer->comp);
1410         spin_lock_irqsave(&master->xferqueue.lock, flags);
1411         if (master->xferqueue.cur) {
1412                 list_add_tail(&xfer->node, &master->xferqueue.list);
1413         } else {
1414                 master->xferqueue.cur = xfer;
1415                 svc_i3c_master_start_xfer_locked(master);
1416         }
1417         spin_unlock_irqrestore(&master->xferqueue.lock, flags);
1418
1419         pm_runtime_mark_last_busy(master->dev);
1420         pm_runtime_put_autosuspend(master->dev);
1421 }
1422
1423 static bool
1424 svc_i3c_master_supports_ccc_cmd(struct i3c_master_controller *master,
1425                                 const struct i3c_ccc_cmd *cmd)
1426 {
1427         /* No software support for CCC commands targeting more than one slave */
1428         return (cmd->ndests == 1);
1429 }
1430
1431 static int svc_i3c_master_send_bdcast_ccc_cmd(struct svc_i3c_master *master,
1432                                               struct i3c_ccc_cmd *ccc)
1433 {
1434         unsigned int xfer_len = ccc->dests[0].payload.len + 1;
1435         struct svc_i3c_xfer *xfer;
1436         struct svc_i3c_cmd *cmd;
1437         u8 *buf;
1438         int ret;
1439
1440         xfer = svc_i3c_master_alloc_xfer(master, 1);
1441         if (!xfer)
1442                 return -ENOMEM;
1443
1444         buf = kmalloc(xfer_len, GFP_KERNEL);
1445         if (!buf) {
1446                 svc_i3c_master_free_xfer(xfer);
1447                 return -ENOMEM;
1448         }
1449
1450         buf[0] = ccc->id;
1451         memcpy(&buf[1], ccc->dests[0].payload.data, ccc->dests[0].payload.len);
1452
1453         xfer->type = SVC_I3C_MCTRL_TYPE_I3C;
1454
1455         cmd = &xfer->cmds[0];
1456         cmd->addr = ccc->dests[0].addr;
1457         cmd->rnw = ccc->rnw;
1458         cmd->in = NULL;
1459         cmd->out = buf;
1460         cmd->len = xfer_len;
1461         cmd->actual_len = 0;
1462         cmd->continued = false;
1463
1464         mutex_lock(&master->lock);
1465         svc_i3c_master_enqueue_xfer(master, xfer);
1466         if (!wait_for_completion_timeout(&xfer->comp, msecs_to_jiffies(1000)))
1467                 svc_i3c_master_dequeue_xfer(master, xfer);
1468         mutex_unlock(&master->lock);
1469
1470         ret = xfer->ret;
1471         kfree(buf);
1472         svc_i3c_master_free_xfer(xfer);
1473
1474         return ret;
1475 }
1476
1477 static int svc_i3c_master_send_direct_ccc_cmd(struct svc_i3c_master *master,
1478                                               struct i3c_ccc_cmd *ccc)
1479 {
1480         unsigned int xfer_len = ccc->dests[0].payload.len;
1481         unsigned int actual_len = ccc->rnw ? xfer_len : 0;
1482         struct svc_i3c_xfer *xfer;
1483         struct svc_i3c_cmd *cmd;
1484         int ret;
1485
1486         xfer = svc_i3c_master_alloc_xfer(master, 2);
1487         if (!xfer)
1488                 return -ENOMEM;
1489
1490         xfer->type = SVC_I3C_MCTRL_TYPE_I3C;
1491
1492         /* Broadcasted message */
1493         cmd = &xfer->cmds[0];
1494         cmd->addr = I3C_BROADCAST_ADDR;
1495         cmd->rnw = 0;
1496         cmd->in = NULL;
1497         cmd->out = &ccc->id;
1498         cmd->len = 1;
1499         cmd->actual_len = 0;
1500         cmd->continued = true;
1501
1502         /* Directed message */
1503         cmd = &xfer->cmds[1];
1504         cmd->addr = ccc->dests[0].addr;
1505         cmd->rnw = ccc->rnw;
1506         cmd->in = ccc->rnw ? ccc->dests[0].payload.data : NULL;
1507         cmd->out = ccc->rnw ? NULL : ccc->dests[0].payload.data;
1508         cmd->len = xfer_len;
1509         cmd->actual_len = actual_len;
1510         cmd->continued = false;
1511
1512         mutex_lock(&master->lock);
1513         svc_i3c_master_enqueue_xfer(master, xfer);
1514         if (!wait_for_completion_timeout(&xfer->comp, msecs_to_jiffies(1000)))
1515                 svc_i3c_master_dequeue_xfer(master, xfer);
1516         mutex_unlock(&master->lock);
1517
1518         if (cmd->actual_len != xfer_len)
1519                 ccc->dests[0].payload.len = cmd->actual_len;
1520
1521         ret = xfer->ret;
1522         svc_i3c_master_free_xfer(xfer);
1523
1524         return ret;
1525 }
1526
1527 static int svc_i3c_master_send_ccc_cmd(struct i3c_master_controller *m,
1528                                        struct i3c_ccc_cmd *cmd)
1529 {
1530         struct svc_i3c_master *master = to_svc_i3c_master(m);
1531         bool broadcast = cmd->id < 0x80;
1532         int ret;
1533
1534         if (broadcast)
1535                 ret = svc_i3c_master_send_bdcast_ccc_cmd(master, cmd);
1536         else
1537                 ret = svc_i3c_master_send_direct_ccc_cmd(master, cmd);
1538
1539         if (ret)
1540                 cmd->err = I3C_ERROR_M2;
1541
1542         return ret;
1543 }
1544
1545 static int svc_i3c_master_priv_xfers(struct i3c_dev_desc *dev,
1546                                      struct i3c_priv_xfer *xfers,
1547                                      int nxfers)
1548 {
1549         struct i3c_master_controller *m = i3c_dev_get_master(dev);
1550         struct svc_i3c_master *master = to_svc_i3c_master(m);
1551         struct svc_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev);
1552         struct svc_i3c_xfer *xfer;
1553         int ret, i;
1554
1555         xfer = svc_i3c_master_alloc_xfer(master, nxfers);
1556         if (!xfer)
1557                 return -ENOMEM;
1558
1559         xfer->type = SVC_I3C_MCTRL_TYPE_I3C;
1560
1561         for (i = 0; i < nxfers; i++) {
1562                 struct svc_i3c_cmd *cmd = &xfer->cmds[i];
1563
1564                 cmd->xfer = &xfers[i];
1565                 cmd->addr = master->addrs[data->index];
1566                 cmd->rnw = xfers[i].rnw;
1567                 cmd->in = xfers[i].rnw ? xfers[i].data.in : NULL;
1568                 cmd->out = xfers[i].rnw ? NULL : xfers[i].data.out;
1569                 cmd->len = xfers[i].len;
1570                 cmd->actual_len = xfers[i].rnw ? xfers[i].len : 0;
1571                 cmd->continued = (i + 1) < nxfers;
1572         }
1573
1574         mutex_lock(&master->lock);
1575         svc_i3c_master_enqueue_xfer(master, xfer);
1576         if (!wait_for_completion_timeout(&xfer->comp, msecs_to_jiffies(1000)))
1577                 svc_i3c_master_dequeue_xfer(master, xfer);
1578         mutex_unlock(&master->lock);
1579
1580         ret = xfer->ret;
1581         svc_i3c_master_free_xfer(xfer);
1582
1583         return ret;
1584 }
1585
1586 static int svc_i3c_master_i2c_xfers(struct i2c_dev_desc *dev,
1587                                     const struct i2c_msg *xfers,
1588                                     int nxfers)
1589 {
1590         struct i3c_master_controller *m = i2c_dev_get_master(dev);
1591         struct svc_i3c_master *master = to_svc_i3c_master(m);
1592         struct svc_i3c_i2c_dev_data *data = i2c_dev_get_master_data(dev);
1593         struct svc_i3c_xfer *xfer;
1594         int ret, i;
1595
1596         xfer = svc_i3c_master_alloc_xfer(master, nxfers);
1597         if (!xfer)
1598                 return -ENOMEM;
1599
1600         xfer->type = SVC_I3C_MCTRL_TYPE_I2C;
1601
1602         for (i = 0; i < nxfers; i++) {
1603                 struct svc_i3c_cmd *cmd = &xfer->cmds[i];
1604
1605                 cmd->addr = master->addrs[data->index];
1606                 cmd->rnw = xfers[i].flags & I2C_M_RD;
1607                 cmd->in = cmd->rnw ? xfers[i].buf : NULL;
1608                 cmd->out = cmd->rnw ? NULL : xfers[i].buf;
1609                 cmd->len = xfers[i].len;
1610                 cmd->actual_len = cmd->rnw ? xfers[i].len : 0;
1611                 cmd->continued = (i + 1 < nxfers);
1612         }
1613
1614         mutex_lock(&master->lock);
1615         svc_i3c_master_enqueue_xfer(master, xfer);
1616         if (!wait_for_completion_timeout(&xfer->comp, msecs_to_jiffies(1000)))
1617                 svc_i3c_master_dequeue_xfer(master, xfer);
1618         mutex_unlock(&master->lock);
1619
1620         ret = xfer->ret;
1621         svc_i3c_master_free_xfer(xfer);
1622
1623         return ret;
1624 }
1625
1626 static int svc_i3c_master_request_ibi(struct i3c_dev_desc *dev,
1627                                       const struct i3c_ibi_setup *req)
1628 {
1629         struct i3c_master_controller *m = i3c_dev_get_master(dev);
1630         struct svc_i3c_master *master = to_svc_i3c_master(m);
1631         struct svc_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev);
1632         unsigned long flags;
1633         unsigned int i;
1634
1635         if (dev->ibi->max_payload_len > SVC_I3C_FIFO_SIZE) {
1636                 dev_err(master->dev, "IBI max payload %d should be < %d\n",
1637                         dev->ibi->max_payload_len, SVC_I3C_FIFO_SIZE);
1638                 return -ERANGE;
1639         }
1640
1641         data->ibi_pool = i3c_generic_ibi_alloc_pool(dev, req);
1642         if (IS_ERR(data->ibi_pool))
1643                 return PTR_ERR(data->ibi_pool);
1644
1645         spin_lock_irqsave(&master->ibi.lock, flags);
1646         for (i = 0; i < master->ibi.num_slots; i++) {
1647                 if (!master->ibi.slots[i]) {
1648                         data->ibi = i;
1649                         master->ibi.slots[i] = dev;
1650                         break;
1651                 }
1652         }
1653         spin_unlock_irqrestore(&master->ibi.lock, flags);
1654
1655         if (i < master->ibi.num_slots)
1656                 return 0;
1657
1658         i3c_generic_ibi_free_pool(data->ibi_pool);
1659         data->ibi_pool = NULL;
1660
1661         return -ENOSPC;
1662 }
1663
1664 static void svc_i3c_master_free_ibi(struct i3c_dev_desc *dev)
1665 {
1666         struct i3c_master_controller *m = i3c_dev_get_master(dev);
1667         struct svc_i3c_master *master = to_svc_i3c_master(m);
1668         struct svc_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev);
1669         unsigned long flags;
1670
1671         spin_lock_irqsave(&master->ibi.lock, flags);
1672         master->ibi.slots[data->ibi] = NULL;
1673         data->ibi = -1;
1674         spin_unlock_irqrestore(&master->ibi.lock, flags);
1675
1676         i3c_generic_ibi_free_pool(data->ibi_pool);
1677 }
1678
1679 static int svc_i3c_master_enable_ibi(struct i3c_dev_desc *dev)
1680 {
1681         struct i3c_master_controller *m = i3c_dev_get_master(dev);
1682         struct svc_i3c_master *master = to_svc_i3c_master(m);
1683         int ret;
1684
1685         ret = pm_runtime_resume_and_get(master->dev);
1686         if (ret < 0) {
1687                 dev_err(master->dev, "<%s> Cannot get runtime PM.\n", __func__);
1688                 return ret;
1689         }
1690
1691         master->enabled_events++;
1692         svc_i3c_master_enable_interrupts(master, SVC_I3C_MINT_SLVSTART);
1693
1694         return i3c_master_enec_locked(m, dev->info.dyn_addr, I3C_CCC_EVENT_SIR);
1695 }
1696
1697 static int svc_i3c_master_disable_ibi(struct i3c_dev_desc *dev)
1698 {
1699         struct i3c_master_controller *m = i3c_dev_get_master(dev);
1700         struct svc_i3c_master *master = to_svc_i3c_master(m);
1701         int ret;
1702
1703         master->enabled_events--;
1704         if (!master->enabled_events)
1705                 svc_i3c_master_disable_interrupts(master);
1706
1707         ret = i3c_master_disec_locked(m, dev->info.dyn_addr, I3C_CCC_EVENT_SIR);
1708
1709         pm_runtime_mark_last_busy(master->dev);
1710         pm_runtime_put_autosuspend(master->dev);
1711
1712         return ret;
1713 }
1714
1715 static int svc_i3c_master_enable_hotjoin(struct i3c_master_controller *m)
1716 {
1717         struct svc_i3c_master *master = to_svc_i3c_master(m);
1718         int ret;
1719
1720         ret = pm_runtime_resume_and_get(master->dev);
1721         if (ret < 0) {
1722                 dev_err(master->dev, "<%s> Cannot get runtime PM.\n", __func__);
1723                 return ret;
1724         }
1725
1726         master->enabled_events |= SVC_I3C_EVENT_HOTJOIN;
1727
1728         svc_i3c_master_enable_interrupts(master, SVC_I3C_MINT_SLVSTART);
1729
1730         return 0;
1731 }
1732
1733 static int svc_i3c_master_disable_hotjoin(struct i3c_master_controller *m)
1734 {
1735         struct svc_i3c_master *master = to_svc_i3c_master(m);
1736
1737         master->enabled_events &= ~SVC_I3C_EVENT_HOTJOIN;
1738
1739         if (!master->enabled_events)
1740                 svc_i3c_master_disable_interrupts(master);
1741
1742         pm_runtime_mark_last_busy(master->dev);
1743         pm_runtime_put_autosuspend(master->dev);
1744
1745         return 0;
1746 }
1747
1748 static void svc_i3c_master_recycle_ibi_slot(struct i3c_dev_desc *dev,
1749                                             struct i3c_ibi_slot *slot)
1750 {
1751         struct svc_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev);
1752
1753         i3c_generic_ibi_recycle_slot(data->ibi_pool, slot);
1754 }
1755
1756 static const struct i3c_master_controller_ops svc_i3c_master_ops = {
1757         .bus_init = svc_i3c_master_bus_init,
1758         .bus_cleanup = svc_i3c_master_bus_cleanup,
1759         .attach_i3c_dev = svc_i3c_master_attach_i3c_dev,
1760         .detach_i3c_dev = svc_i3c_master_detach_i3c_dev,
1761         .reattach_i3c_dev = svc_i3c_master_reattach_i3c_dev,
1762         .attach_i2c_dev = svc_i3c_master_attach_i2c_dev,
1763         .detach_i2c_dev = svc_i3c_master_detach_i2c_dev,
1764         .do_daa = svc_i3c_master_do_daa,
1765         .supports_ccc_cmd = svc_i3c_master_supports_ccc_cmd,
1766         .send_ccc_cmd = svc_i3c_master_send_ccc_cmd,
1767         .priv_xfers = svc_i3c_master_priv_xfers,
1768         .i2c_xfers = svc_i3c_master_i2c_xfers,
1769         .request_ibi = svc_i3c_master_request_ibi,
1770         .free_ibi = svc_i3c_master_free_ibi,
1771         .recycle_ibi_slot = svc_i3c_master_recycle_ibi_slot,
1772         .enable_ibi = svc_i3c_master_enable_ibi,
1773         .disable_ibi = svc_i3c_master_disable_ibi,
1774         .enable_hotjoin = svc_i3c_master_enable_hotjoin,
1775         .disable_hotjoin = svc_i3c_master_disable_hotjoin,
1776         .set_speed = svc_i3c_master_set_speed,
1777 };
1778
1779 static int svc_i3c_master_prepare_clks(struct svc_i3c_master *master)
1780 {
1781         int ret = 0;
1782
1783         ret = clk_prepare_enable(master->pclk);
1784         if (ret)
1785                 return ret;
1786
1787         ret = clk_prepare_enable(master->fclk);
1788         if (ret) {
1789                 clk_disable_unprepare(master->pclk);
1790                 return ret;
1791         }
1792
1793         ret = clk_prepare_enable(master->sclk);
1794         if (ret) {
1795                 clk_disable_unprepare(master->pclk);
1796                 clk_disable_unprepare(master->fclk);
1797                 return ret;
1798         }
1799
1800         return 0;
1801 }
1802
1803 static void svc_i3c_master_unprepare_clks(struct svc_i3c_master *master)
1804 {
1805         clk_disable_unprepare(master->pclk);
1806         clk_disable_unprepare(master->fclk);
1807         clk_disable_unprepare(master->sclk);
1808 }
1809
1810 static int svc_i3c_master_probe(struct platform_device *pdev)
1811 {
1812         struct device *dev = &pdev->dev;
1813         struct svc_i3c_master *master;
1814         int ret;
1815
1816         master = devm_kzalloc(dev, sizeof(*master), GFP_KERNEL);
1817         if (!master)
1818                 return -ENOMEM;
1819
1820         master->regs = devm_platform_ioremap_resource(pdev, 0);
1821         if (IS_ERR(master->regs))
1822                 return PTR_ERR(master->regs);
1823
1824         master->pclk = devm_clk_get(dev, "pclk");
1825         if (IS_ERR(master->pclk))
1826                 return PTR_ERR(master->pclk);
1827
1828         master->fclk = devm_clk_get(dev, "fast_clk");
1829         if (IS_ERR(master->fclk))
1830                 return PTR_ERR(master->fclk);
1831
1832         master->sclk = devm_clk_get(dev, "slow_clk");
1833         if (IS_ERR(master->sclk))
1834                 return PTR_ERR(master->sclk);
1835
1836         master->irq = platform_get_irq(pdev, 0);
1837         if (master->irq < 0)
1838                 return master->irq;
1839
1840         master->dev = dev;
1841
1842         ret = svc_i3c_master_prepare_clks(master);
1843         if (ret)
1844                 return ret;
1845
1846         INIT_WORK(&master->hj_work, svc_i3c_master_hj_work);
1847         INIT_WORK(&master->ibi_work, svc_i3c_master_ibi_work);
1848         mutex_init(&master->lock);
1849
1850         ret = devm_request_irq(dev, master->irq, svc_i3c_master_irq_handler,
1851                                IRQF_NO_SUSPEND, "svc-i3c-irq", master);
1852         if (ret)
1853                 goto err_disable_clks;
1854
1855         master->free_slots = GENMASK(SVC_I3C_MAX_DEVS - 1, 0);
1856
1857         spin_lock_init(&master->xferqueue.lock);
1858         INIT_LIST_HEAD(&master->xferqueue.list);
1859
1860         spin_lock_init(&master->ibi.lock);
1861         master->ibi.num_slots = SVC_I3C_MAX_DEVS;
1862         master->ibi.slots = devm_kcalloc(&pdev->dev, master->ibi.num_slots,
1863                                          sizeof(*master->ibi.slots),
1864                                          GFP_KERNEL);
1865         if (!master->ibi.slots) {
1866                 ret = -ENOMEM;
1867                 goto err_disable_clks;
1868         }
1869
1870         platform_set_drvdata(pdev, master);
1871
1872         pm_runtime_set_autosuspend_delay(&pdev->dev, SVC_I3C_PM_TIMEOUT_MS);
1873         pm_runtime_use_autosuspend(&pdev->dev);
1874         pm_runtime_get_noresume(&pdev->dev);
1875         pm_runtime_set_active(&pdev->dev);
1876         pm_runtime_enable(&pdev->dev);
1877
1878         svc_i3c_master_reset(master);
1879
1880         /* Register the master */
1881         ret = i3c_master_register(&master->base, &pdev->dev,
1882                                   &svc_i3c_master_ops, false);
1883         if (ret)
1884                 goto rpm_disable;
1885
1886         pm_runtime_mark_last_busy(&pdev->dev);
1887         pm_runtime_put_autosuspend(&pdev->dev);
1888
1889         return 0;
1890
1891 rpm_disable:
1892         pm_runtime_dont_use_autosuspend(&pdev->dev);
1893         pm_runtime_put_noidle(&pdev->dev);
1894         pm_runtime_disable(&pdev->dev);
1895         pm_runtime_set_suspended(&pdev->dev);
1896
1897 err_disable_clks:
1898         svc_i3c_master_unprepare_clks(master);
1899
1900         return ret;
1901 }
1902
1903 static void svc_i3c_master_remove(struct platform_device *pdev)
1904 {
1905         struct svc_i3c_master *master = platform_get_drvdata(pdev);
1906
1907         cancel_work_sync(&master->hj_work);
1908         i3c_master_unregister(&master->base);
1909
1910         pm_runtime_dont_use_autosuspend(&pdev->dev);
1911         pm_runtime_disable(&pdev->dev);
1912 }
1913
1914 static void svc_i3c_save_regs(struct svc_i3c_master *master)
1915 {
1916         master->saved_regs.mconfig = readl(master->regs + SVC_I3C_MCONFIG);
1917         master->saved_regs.mdynaddr = readl(master->regs + SVC_I3C_MDYNADDR);
1918 }
1919
1920 static void svc_i3c_restore_regs(struct svc_i3c_master *master)
1921 {
1922         if (readl(master->regs + SVC_I3C_MDYNADDR) !=
1923             master->saved_regs.mdynaddr) {
1924                 writel(master->saved_regs.mconfig,
1925                        master->regs + SVC_I3C_MCONFIG);
1926                 writel(master->saved_regs.mdynaddr,
1927                        master->regs + SVC_I3C_MDYNADDR);
1928         }
1929 }
1930
1931 static int __maybe_unused svc_i3c_runtime_suspend(struct device *dev)
1932 {
1933         struct svc_i3c_master *master = dev_get_drvdata(dev);
1934
1935         svc_i3c_save_regs(master);
1936         svc_i3c_master_unprepare_clks(master);
1937         pinctrl_pm_select_sleep_state(dev);
1938
1939         return 0;
1940 }
1941
1942 static int __maybe_unused svc_i3c_runtime_resume(struct device *dev)
1943 {
1944         struct svc_i3c_master *master = dev_get_drvdata(dev);
1945
1946         pinctrl_pm_select_default_state(dev);
1947         svc_i3c_master_prepare_clks(master);
1948
1949         svc_i3c_restore_regs(master);
1950
1951         return 0;
1952 }
1953
1954 static const struct dev_pm_ops svc_i3c_pm_ops = {
1955         SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
1956                                       pm_runtime_force_resume)
1957         SET_RUNTIME_PM_OPS(svc_i3c_runtime_suspend,
1958                            svc_i3c_runtime_resume, NULL)
1959 };
1960
1961 static const struct of_device_id svc_i3c_master_of_match_tbl[] = {
1962         { .compatible = "silvaco,i3c-master-v1"},
1963         { /* sentinel */ },
1964 };
1965 MODULE_DEVICE_TABLE(of, svc_i3c_master_of_match_tbl);
1966
1967 static struct platform_driver svc_i3c_master = {
1968         .probe = svc_i3c_master_probe,
1969         .remove = svc_i3c_master_remove,
1970         .driver = {
1971                 .name = "silvaco-i3c-master",
1972                 .of_match_table = svc_i3c_master_of_match_tbl,
1973                 .pm = &svc_i3c_pm_ops,
1974         },
1975 };
1976 module_platform_driver(svc_i3c_master);
1977
1978 MODULE_AUTHOR("Conor Culhane <[email protected]>");
1979 MODULE_AUTHOR("Miquel Raynal <[email protected]>");
1980 MODULE_DESCRIPTION("Silvaco dual-role I3C master driver");
1981 MODULE_LICENSE("GPL v2");