drivers/net/ethernet/freescale/fman/fman.c

   1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0-or-later
   2 /*
   3  * Copyright 2008 - 2015 Freescale Semiconductor Inc.
   4  * Copyright 2020 NXP
   5  */
   6
   7 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   8
   9 #include <linux/fsl/guts.h>
  10 #include <linux/platform_device.h>
  11 #include <linux/slab.h>
  12 #include <linux/delay.h>
  13 #include <linux/module.h>
  14 #include <linux/of_platform.h>
  15 #include <linux/clk.h>
  16 #include <linux/of_address.h>
  17 #include <linux/of_irq.h>
  18 #include <linux/interrupt.h>
  19 #include <linux/libfdt_env.h>
  20
  21 #include "fman.h"
  22 #include "fman_muram.h"
  23 #include "fman_keygen.h"
  24
  25 /* General defines */
  26 #define FMAN_LIODN_TBL                  64      /* size of LIODN table */
  27 #define FM_NUM_OF_FMAN_CTRL_EVENT_REGS  4
  28 #define BASE_RX_PORTID                  0x08
  29 #define BASE_TX_PORTID                  0x28
  30
  31 /* Modules registers offsets */
  32 #define BMI_OFFSET              0x00080000
  33 #define QMI_OFFSET              0x00080400
  34 #define KG_OFFSET               0x000C1000
  35 #define DMA_OFFSET              0x000C2000
  36 #define FPM_OFFSET              0x000C3000
  37 #define IMEM_OFFSET             0x000C4000
  38 #define HWP_OFFSET              0x000C7000
  39 #define CGP_OFFSET              0x000DB000
  40
  41 /* Exceptions bit map */
  42 #define EX_DMA_BUS_ERROR                0x80000000
  43 #define EX_DMA_READ_ECC                 0x40000000
  44 #define EX_DMA_SYSTEM_WRITE_ECC 0x20000000
  45 #define EX_DMA_FM_WRITE_ECC             0x10000000
  46 #define EX_FPM_STALL_ON_TASKS           0x08000000
  47 #define EX_FPM_SINGLE_ECC               0x04000000
  48 #define EX_FPM_DOUBLE_ECC               0x02000000
  49 #define EX_QMI_SINGLE_ECC               0x01000000
  50 #define EX_QMI_DEQ_FROM_UNKNOWN_PORTID  0x00800000
  51 #define EX_QMI_DOUBLE_ECC               0x00400000
  52 #define EX_BMI_LIST_RAM_ECC             0x00200000
  53 #define EX_BMI_STORAGE_PROFILE_ECC      0x00100000
  54 #define EX_BMI_STATISTICS_RAM_ECC       0x00080000
  55 #define EX_IRAM_ECC                     0x00040000
  56 #define EX_MURAM_ECC                    0x00020000
  57 #define EX_BMI_DISPATCH_RAM_ECC 0x00010000
  58 #define EX_DMA_SINGLE_PORT_ECC          0x00008000
  59
  60 /* DMA defines */
  61 /* masks */
  62 #define DMA_MODE_BER                    0x00200000
  63 #define DMA_MODE_ECC                    0x00000020
  64 #define DMA_MODE_SECURE_PROT            0x00000800
  65 #define DMA_MODE_AXI_DBG_MASK           0x0F000000
  66
  67 #define DMA_TRANSFER_PORTID_MASK        0xFF000000
  68 #define DMA_TRANSFER_TNUM_MASK          0x00FF0000
  69 #define DMA_TRANSFER_LIODN_MASK 0x00000FFF
  70
  71 #define DMA_STATUS_BUS_ERR              0x08000000
  72 #define DMA_STATUS_READ_ECC             0x04000000
  73 #define DMA_STATUS_SYSTEM_WRITE_ECC     0x02000000
  74 #define DMA_STATUS_FM_WRITE_ECC 0x01000000
  75 #define DMA_STATUS_FM_SPDAT_ECC 0x00080000
  76
  77 #define DMA_MODE_CACHE_OR_SHIFT         30
  78 #define DMA_MODE_AXI_DBG_SHIFT                  24
  79 #define DMA_MODE_CEN_SHIFT                      13
  80 #define DMA_MODE_CEN_MASK                       0x00000007
  81 #define DMA_MODE_DBG_SHIFT                      7
  82 #define DMA_MODE_AID_MODE_SHIFT         4
  83
  84 #define DMA_THRESH_COMMQ_SHIFT                  24
  85 #define DMA_THRESH_READ_INT_BUF_SHIFT           16
  86 #define DMA_THRESH_READ_INT_BUF_MASK            0x0000003f
  87 #define DMA_THRESH_WRITE_INT_BUF_MASK           0x0000003f
  88
  89 #define DMA_TRANSFER_PORTID_SHIFT               24
  90 #define DMA_TRANSFER_TNUM_SHIFT         16
  91
  92 #define DMA_CAM_SIZEOF_ENTRY                    0x40
  93 #define DMA_CAM_UNITS                           8
  94
  95 #define DMA_LIODN_SHIFT         16
  96 #define DMA_LIODN_BASE_MASK     0x00000FFF
  97
  98 /* FPM defines */
  99 #define FPM_EV_MASK_DOUBLE_ECC          0x80000000
 100 #define FPM_EV_MASK_STALL               0x40000000
 101 #define FPM_EV_MASK_SINGLE_ECC          0x20000000
 102 #define FPM_EV_MASK_RELEASE_FM          0x00010000
 103 #define FPM_EV_MASK_DOUBLE_ECC_EN       0x00008000
 104 #define FPM_EV_MASK_STALL_EN            0x00004000
 105 #define FPM_EV_MASK_SINGLE_ECC_EN       0x00002000
 106 #define FPM_EV_MASK_EXTERNAL_HALT       0x00000008
 107 #define FPM_EV_MASK_ECC_ERR_HALT        0x00000004
 108
 109 #define FPM_RAM_MURAM_ECC               0x00008000
 110 #define FPM_RAM_IRAM_ECC                0x00004000
 111 #define FPM_IRAM_ECC_ERR_EX_EN          0x00020000
 112 #define FPM_MURAM_ECC_ERR_EX_EN 0x00040000
 113 #define FPM_RAM_IRAM_ECC_EN             0x40000000
 114 #define FPM_RAM_RAMS_ECC_EN             0x80000000
 115 #define FPM_RAM_RAMS_ECC_EN_SRC_SEL     0x08000000
 116
 117 #define FPM_REV1_MAJOR_MASK             0x0000FF00
 118 #define FPM_REV1_MINOR_MASK             0x000000FF
 119
 120 #define FPM_DISP_LIMIT_SHIFT            24
 121
 122 #define FPM_PRT_FM_CTL1                 0x00000001
 123 #define FPM_PRT_FM_CTL2                 0x00000002
 124 #define FPM_PORT_FM_CTL_PORTID_SHIFT    24
 125 #define FPM_PRC_ORA_FM_CTL_SEL_SHIFT    16
 126
 127 #define FPM_THR1_PRS_SHIFT              24
 128 #define FPM_THR1_KG_SHIFT               16
 129 #define FPM_THR1_PLCR_SHIFT             8
 130 #define FPM_THR1_BMI_SHIFT              0
 131
 132 #define FPM_THR2_QMI_ENQ_SHIFT          24
 133 #define FPM_THR2_QMI_DEQ_SHIFT          0
 134 #define FPM_THR2_FM_CTL1_SHIFT          16
 135 #define FPM_THR2_FM_CTL2_SHIFT          8
 136
 137 #define FPM_EV_MASK_CAT_ERR_SHIFT       1
 138 #define FPM_EV_MASK_DMA_ERR_SHIFT       0
 139
 140 #define FPM_REV1_MAJOR_SHIFT            8
 141
 142 #define FPM_RSTC_FM_RESET               0x80000000
 143 #define FPM_RSTC_MAC0_RESET             0x40000000
 144 #define FPM_RSTC_MAC1_RESET             0x20000000
 145 #define FPM_RSTC_MAC2_RESET             0x10000000
 146 #define FPM_RSTC_MAC3_RESET             0x08000000
 147 #define FPM_RSTC_MAC8_RESET             0x04000000
 148 #define FPM_RSTC_MAC4_RESET             0x02000000
 149 #define FPM_RSTC_MAC5_RESET             0x01000000
 150 #define FPM_RSTC_MAC6_RESET             0x00800000
 151 #define FPM_RSTC_MAC7_RESET             0x00400000
 152 #define FPM_RSTC_MAC9_RESET             0x00200000
 153
 154 #define FPM_TS_INT_SHIFT                16
 155 #define FPM_TS_CTL_EN                   0x80000000
 156
 157 /* BMI defines */
 158 #define BMI_INIT_START                          0x80000000
 159 #define BMI_ERR_INTR_EN_STORAGE_PROFILE_ECC     0x80000000
 160 #define BMI_ERR_INTR_EN_LIST_RAM_ECC            0x40000000
 161 #define BMI_ERR_INTR_EN_STATISTICS_RAM_ECC      0x20000000
 162 #define BMI_ERR_INTR_EN_DISPATCH_RAM_ECC        0x10000000
 163 #define BMI_NUM_OF_TASKS_MASK                   0x3F000000
 164 #define BMI_NUM_OF_EXTRA_TASKS_MASK             0x000F0000
 165 #define BMI_NUM_OF_DMAS_MASK                    0x00000F00
 166 #define BMI_NUM_OF_EXTRA_DMAS_MASK              0x0000000F
 167 #define BMI_FIFO_SIZE_MASK                      0x000003FF
 168 #define BMI_EXTRA_FIFO_SIZE_MASK                0x03FF0000
 169 #define BMI_CFG2_DMAS_MASK                      0x0000003F
 170 #define BMI_CFG2_TASKS_MASK                     0x0000003F
 171
 172 #define BMI_CFG2_TASKS_SHIFT            16
 173 #define BMI_CFG2_DMAS_SHIFT             0
 174 #define BMI_CFG1_FIFO_SIZE_SHIFT        16
 175 #define BMI_NUM_OF_TASKS_SHIFT          24
 176 #define BMI_EXTRA_NUM_OF_TASKS_SHIFT    16
 177 #define BMI_NUM_OF_DMAS_SHIFT           8
 178 #define BMI_EXTRA_NUM_OF_DMAS_SHIFT     0
 179
 180 #define BMI_FIFO_ALIGN                  0x100
 181
 182 #define BMI_EXTRA_FIFO_SIZE_SHIFT       16
 183
 184 /* QMI defines */
 185 #define QMI_CFG_ENQ_EN                  0x80000000
 186 #define QMI_CFG_DEQ_EN                  0x40000000
 187 #define QMI_CFG_EN_COUNTERS             0x10000000
 188 #define QMI_CFG_DEQ_MASK                0x0000003F
 189 #define QMI_CFG_ENQ_MASK                0x00003F00
 190 #define QMI_CFG_ENQ_SHIFT               8
 191
 192 #define QMI_ERR_INTR_EN_DOUBLE_ECC      0x80000000
 193 #define QMI_ERR_INTR_EN_DEQ_FROM_DEF    0x40000000
 194 #define QMI_INTR_EN_SINGLE_ECC          0x80000000
 195
 196 #define QMI_GS_HALT_NOT_BUSY            0x00000002
 197
 198 /* HWP defines */
 199 #define HWP_RPIMAC_PEN                  0x00000001
 200
 201 /* IRAM defines */
 202 #define IRAM_IADD_AIE                   0x80000000
 203 #define IRAM_READY                      0x80000000
 204
 205 /* Default values */
 206 #define DEFAULT_CATASTROPHIC_ERR                0
 207 #define DEFAULT_DMA_ERR                         0
 208 #define DEFAULT_AID_MODE                        FMAN_DMA_AID_OUT_TNUM
 209 #define DEFAULT_DMA_COMM_Q_LOW                  0x2A
 210 #define DEFAULT_DMA_COMM_Q_HIGH         0x3F
 211 #define DEFAULT_CACHE_OVERRIDE                  0
 212 #define DEFAULT_DMA_CAM_NUM_OF_ENTRIES          64
 213 #define DEFAULT_DMA_DBG_CNT_MODE                0
 214 #define DEFAULT_DMA_SOS_EMERGENCY               0
 215 #define DEFAULT_DMA_WATCHDOG                    0
 216 #define DEFAULT_DISP_LIMIT                      0
 217 #define DEFAULT_PRS_DISP_TH                     16
 218 #define DEFAULT_PLCR_DISP_TH                    16
 219 #define DEFAULT_KG_DISP_TH                      16
 220 #define DEFAULT_BMI_DISP_TH                     16
 221 #define DEFAULT_QMI_ENQ_DISP_TH         16
 222 #define DEFAULT_QMI_DEQ_DISP_TH         16
 223 #define DEFAULT_FM_CTL1_DISP_TH         16
 224 #define DEFAULT_FM_CTL2_DISP_TH         16
 225
 226 #define DFLT_AXI_DBG_NUM_OF_BEATS               1
 227
 228 #define DFLT_DMA_READ_INT_BUF_LOW(dma_thresh_max_buf)   \
 229         ((dma_thresh_max_buf + 1) / 2)
 230 #define DFLT_DMA_READ_INT_BUF_HIGH(dma_thresh_max_buf)  \
 231         ((dma_thresh_max_buf + 1) * 3 / 4)
 232 #define DFLT_DMA_WRITE_INT_BUF_LOW(dma_thresh_max_buf)  \
 233         ((dma_thresh_max_buf + 1) / 2)
 234 #define DFLT_DMA_WRITE_INT_BUF_HIGH(dma_thresh_max_buf)\
 235         ((dma_thresh_max_buf + 1) * 3 / 4)
 236
 237 #define DMA_COMM_Q_LOW_FMAN_V3          0x2A
 238 #define DMA_COMM_Q_LOW_FMAN_V2(dma_thresh_max_commq)            \
 239         ((dma_thresh_max_commq + 1) / 2)
 240 #define DFLT_DMA_COMM_Q_LOW(major, dma_thresh_max_commq)        \
 241         ((major == 6) ? DMA_COMM_Q_LOW_FMAN_V3 :                \
 242         DMA_COMM_Q_LOW_FMAN_V2(dma_thresh_max_commq))
 243
 244 #define DMA_COMM_Q_HIGH_FMAN_V3 0x3f
 245 #define DMA_COMM_Q_HIGH_FMAN_V2(dma_thresh_max_commq)           \
 246         ((dma_thresh_max_commq + 1) * 3 / 4)
 247 #define DFLT_DMA_COMM_Q_HIGH(major, dma_thresh_max_commq)       \
 248         ((major == 6) ? DMA_COMM_Q_HIGH_FMAN_V3 :               \
 249         DMA_COMM_Q_HIGH_FMAN_V2(dma_thresh_max_commq))
 250
 251 #define TOTAL_NUM_OF_TASKS_FMAN_V3L     59
 252 #define TOTAL_NUM_OF_TASKS_FMAN_V3H     124
 253 #define DFLT_TOTAL_NUM_OF_TASKS(major, minor, bmi_max_num_of_tasks)     \
 254         ((major == 6) ? ((minor == 1 || minor == 4) ?                   \
 255         TOTAL_NUM_OF_TASKS_FMAN_V3L : TOTAL_NUM_OF_TASKS_FMAN_V3H) :    \
 256         bmi_max_num_of_tasks)
 257
 258 #define DMA_CAM_NUM_OF_ENTRIES_FMAN_V3          64
 259 #define DMA_CAM_NUM_OF_ENTRIES_FMAN_V2          32
 260 #define DFLT_DMA_CAM_NUM_OF_ENTRIES(major)                      \
 261         (major == 6 ? DMA_CAM_NUM_OF_ENTRIES_FMAN_V3 :          \
 262         DMA_CAM_NUM_OF_ENTRIES_FMAN_V2)
 263
 264 #define FM_TIMESTAMP_1_USEC_BIT             8
 265
 266 /* Defines used for enabling/disabling FMan interrupts */
 267 #define ERR_INTR_EN_DMA         0x00010000
 268 #define ERR_INTR_EN_FPM         0x80000000
 269 #define ERR_INTR_EN_BMI         0x00800000
 270 #define ERR_INTR_EN_QMI         0x00400000
 271 #define ERR_INTR_EN_MURAM       0x00040000
 272 #define ERR_INTR_EN_MAC0        0x00004000
 273 #define ERR_INTR_EN_MAC1        0x00002000
 274 #define ERR_INTR_EN_MAC2        0x00001000
 275 #define ERR_INTR_EN_MAC3        0x00000800
 276 #define ERR_INTR_EN_MAC4        0x00000400
 277 #define ERR_INTR_EN_MAC5        0x00000200
 278 #define ERR_INTR_EN_MAC6        0x00000100
 279 #define ERR_INTR_EN_MAC7        0x00000080
 280 #define ERR_INTR_EN_MAC8        0x00008000
 281 #define ERR_INTR_EN_MAC9        0x00000040
 282
 283 #define INTR_EN_QMI             0x40000000
 284 #define INTR_EN_MAC0            0x00080000
 285 #define INTR_EN_MAC1            0x00040000
 286 #define INTR_EN_MAC2            0x00020000
 287 #define INTR_EN_MAC3            0x00010000
 288 #define INTR_EN_MAC4            0x00000040
 289 #define INTR_EN_MAC5            0x00000020
 290 #define INTR_EN_MAC6            0x00000008
 291 #define INTR_EN_MAC7            0x00000002
 292 #define INTR_EN_MAC8            0x00200000
 293 #define INTR_EN_MAC9            0x00100000
 294 #define INTR_EN_REV0            0x00008000
 295 #define INTR_EN_REV1            0x00004000
 296 #define INTR_EN_REV2            0x00002000
 297 #define INTR_EN_REV3            0x00001000
 298 #define INTR_EN_TMR             0x01000000
 299
 300 enum fman_dma_aid_mode {
 301         FMAN_DMA_AID_OUT_PORT_ID = 0,             /* 4 LSB of PORT_ID */
 302         FMAN_DMA_AID_OUT_TNUM                     /* 4 LSB of TNUM */
 303 };
 304
 305 struct fman_iram_regs {
 306         u32 iadd;       /* FM IRAM instruction address register */
 307         u32 idata;      /* FM IRAM instruction data register */
 308         u32 itcfg;      /* FM IRAM timing config register */
 309         u32 iready;     /* FM IRAM ready register */
 310 };
 311
 312 struct fman_fpm_regs {
 313         u32 fmfp_tnc;           /* FPM TNUM Control 0x00 */
 314         u32 fmfp_prc;           /* FPM Port_ID FmCtl Association 0x04 */
 315         u32 fmfp_brkc;          /* FPM Breakpoint Control 0x08 */
 316         u32 fmfp_mxd;           /* FPM Flush Control 0x0c */
 317         u32 fmfp_dist1;         /* FPM Dispatch Thresholds1 0x10 */
 318         u32 fmfp_dist2;         /* FPM Dispatch Thresholds2 0x14 */
 319         u32 fm_epi;             /* FM Error Pending Interrupts 0x18 */
 320         u32 fm_rie;             /* FM Error Interrupt Enable 0x1c */
 321         u32 fmfp_fcev[4];       /* FPM FMan-Controller Event 1-4 0x20-0x2f */
 322         u32 res0030[4];         /* res 0x30 - 0x3f */
 323         u32 fmfp_cee[4];        /* PM FMan-Controller Event 1-4 0x40-0x4f */
 324         u32 res0050[4];         /* res 0x50-0x5f */
 325         u32 fmfp_tsc1;          /* FPM TimeStamp Control1 0x60 */
 326         u32 fmfp_tsc2;          /* FPM TimeStamp Control2 0x64 */
 327         u32 fmfp_tsp;           /* FPM Time Stamp 0x68 */
 328         u32 fmfp_tsf;           /* FPM Time Stamp Fraction 0x6c */
 329         u32 fm_rcr;             /* FM Rams Control 0x70 */
 330         u32 fmfp_extc;          /* FPM External Requests Control 0x74 */
 331         u32 fmfp_ext1;          /* FPM External Requests Config1 0x78 */
 332         u32 fmfp_ext2;          /* FPM External Requests Config2 0x7c */
 333         u32 fmfp_drd[16];       /* FPM Data_Ram Data 0-15 0x80 - 0xbf */
 334         u32 fmfp_dra;           /* FPM Data Ram Access 0xc0 */
 335         u32 fm_ip_rev_1;        /* FM IP Block Revision 1 0xc4 */
 336         u32 fm_ip_rev_2;        /* FM IP Block Revision 2 0xc8 */
 337         u32 fm_rstc;            /* FM Reset Command 0xcc */
 338         u32 fm_cld;             /* FM Classifier Debug 0xd0 */
 339         u32 fm_npi;             /* FM Normal Pending Interrupts 0xd4 */
 340         u32 fmfp_exte;          /* FPM External Requests Enable 0xd8 */
 341         u32 fmfp_ee;            /* FPM Event&Mask 0xdc */
 342         u32 fmfp_cev[4];        /* FPM CPU Event 1-4 0xe0-0xef */
 343         u32 res00f0[4];         /* res 0xf0-0xff */
 344         u32 fmfp_ps[50];        /* FPM Port Status 0x100-0x1c7 */
 345         u32 res01c8[14];        /* res 0x1c8-0x1ff */
 346         u32 fmfp_clfabc;        /* FPM CLFABC 0x200 */
 347         u32 fmfp_clfcc;         /* FPM CLFCC 0x204 */
 348         u32 fmfp_clfaval;       /* FPM CLFAVAL 0x208 */
 349         u32 fmfp_clfbval;       /* FPM CLFBVAL 0x20c */
 350         u32 fmfp_clfcval;       /* FPM CLFCVAL 0x210 */
 351         u32 fmfp_clfamsk;       /* FPM CLFAMSK 0x214 */
 352         u32 fmfp_clfbmsk;       /* FPM CLFBMSK 0x218 */
 353         u32 fmfp_clfcmsk;       /* FPM CLFCMSK 0x21c */
 354         u32 fmfp_clfamc;        /* FPM CLFAMC 0x220 */
 355         u32 fmfp_clfbmc;        /* FPM CLFBMC 0x224 */
 356         u32 fmfp_clfcmc;        /* FPM CLFCMC 0x228 */
 357         u32 fmfp_decceh;        /* FPM DECCEH 0x22c */
 358         u32 res0230[116];       /* res 0x230 - 0x3ff */
 359         u32 fmfp_ts[128];       /* 0x400: FPM Task Status 0x400 - 0x5ff */
 360         u32 res0600[0x400 - 384];
 361 };
 362
 363 struct fman_bmi_regs {
 364         u32 fmbm_init;          /* BMI Initialization 0x00 */
 365         u32 fmbm_cfg1;          /* BMI Configuration 1 0x04 */
 366         u32 fmbm_cfg2;          /* BMI Configuration 2 0x08 */
 367         u32 res000c[5];         /* 0x0c - 0x1f */
 368         u32 fmbm_ievr;          /* Interrupt Event Register 0x20 */
 369         u32 fmbm_ier;           /* Interrupt Enable Register 0x24 */
 370         u32 fmbm_ifr;           /* Interrupt Force Register 0x28 */
 371         u32 res002c[5];         /* 0x2c - 0x3f */
 372         u32 fmbm_arb[8];        /* BMI Arbitration 0x40 - 0x5f */
 373         u32 res0060[12];        /* 0x60 - 0x8f */
 374         u32 fmbm_dtc[3];        /* Debug Trap Counter 0x90 - 0x9b */
 375         u32 res009c;            /* 0x9c */
 376         u32 fmbm_dcv[3][4];     /* Debug Compare val 0xa0-0xcf */
 377         u32 fmbm_dcm[3][4];     /* Debug Compare Mask 0xd0-0xff */
 378         u32 fmbm_gde;           /* BMI Global Debug Enable 0x100 */
 379         u32 fmbm_pp[63];        /* BMI Port Parameters 0x104 - 0x1ff */
 380         u32 res0200;            /* 0x200 */
 381         u32 fmbm_pfs[63];       /* BMI Port FIFO Size 0x204 - 0x2ff */
 382         u32 res0300;            /* 0x300 */
 383         u32 fmbm_spliodn[63];   /* Port Partition ID 0x304 - 0x3ff */
 384 };
 385
 386 struct fman_qmi_regs {
 387         u32 fmqm_gc;            /* General Configuration Register 0x00 */
 388         u32 res0004;            /* 0x04 */
 389         u32 fmqm_eie;           /* Error Interrupt Event Register 0x08 */
 390         u32 fmqm_eien;          /* Error Interrupt Enable Register 0x0c */
 391         u32 fmqm_eif;           /* Error Interrupt Force Register 0x10 */
 392         u32 fmqm_ie;            /* Interrupt Event Register 0x14 */
 393         u32 fmqm_ien;           /* Interrupt Enable Register 0x18 */
 394         u32 fmqm_if;            /* Interrupt Force Register 0x1c */
 395         u32 fmqm_gs;            /* Global Status Register 0x20 */
 396         u32 fmqm_ts;            /* Task Status Register 0x24 */
 397         u32 fmqm_etfc;          /* Enqueue Total Frame Counter 0x28 */
 398         u32 fmqm_dtfc;          /* Dequeue Total Frame Counter 0x2c */
 399         u32 fmqm_dc0;           /* Dequeue Counter 0 0x30 */
 400         u32 fmqm_dc1;           /* Dequeue Counter 1 0x34 */
 401         u32 fmqm_dc2;           /* Dequeue Counter 2 0x38 */
 402         u32 fmqm_dc3;           /* Dequeue Counter 3 0x3c */
 403         u32 fmqm_dfdc;          /* Dequeue FQID from Default Counter 0x40 */
 404         u32 fmqm_dfcc;          /* Dequeue FQID from Context Counter 0x44 */
 405         u32 fmqm_dffc;          /* Dequeue FQID from FD Counter 0x48 */
 406         u32 fmqm_dcc;           /* Dequeue Confirm Counter 0x4c */
 407         u32 res0050[7];         /* 0x50 - 0x6b */
 408         u32 fmqm_tapc;          /* Tnum Aging Period Control 0x6c */
 409         u32 fmqm_dmcvc;         /* Dequeue MAC Command Valid Counter 0x70 */
 410         u32 fmqm_difdcc;        /* Dequeue Invalid FD Command Counter 0x74 */
 411         u32 fmqm_da1v;          /* Dequeue A1 Valid Counter 0x78 */
 412         u32 res007c;            /* 0x7c */
 413         u32 fmqm_dtc;           /* 0x80 Debug Trap Counter 0x80 */
 414         u32 fmqm_efddd;         /* 0x84 Enqueue Frame desc Dynamic dbg 0x84 */
 415         u32 res0088[2];         /* 0x88 - 0x8f */
 416         struct {
 417                 u32 fmqm_dtcfg1;        /* 0x90 dbg trap cfg 1 Register 0x00 */
 418                 u32 fmqm_dtval1;        /* Debug Trap Value 1 Register 0x04 */
 419                 u32 fmqm_dtm1;          /* Debug Trap Mask 1 Register 0x08 */
 420                 u32 fmqm_dtc1;          /* Debug Trap Counter 1 Register 0x0c */
 421                 u32 fmqm_dtcfg2;        /* dbg Trap cfg 2 Register 0x10 */
 422                 u32 fmqm_dtval2;        /* Debug Trap Value 2 Register 0x14 */
 423                 u32 fmqm_dtm2;          /* Debug Trap Mask 2 Register 0x18 */
 424                 u32 res001c;            /* 0x1c */
 425         } dbg_traps[3];                 /* 0x90 - 0xef */
 426         u8 res00f0[0x400 - 0xf0];       /* 0xf0 - 0x3ff */
 427 };
 428
 429 struct fman_dma_regs {
 430         u32 fmdmsr;     /* FM DMA status register 0x00 */
 431         u32 fmdmmr;     /* FM DMA mode register 0x04 */
 432         u32 fmdmtr;     /* FM DMA bus threshold register 0x08 */
 433         u32 fmdmhy;     /* FM DMA bus hysteresis register 0x0c */
 434         u32 fmdmsetr;   /* FM DMA SOS emergency Threshold Register 0x10 */
 435         u32 fmdmtah;    /* FM DMA transfer bus address high reg 0x14 */
 436         u32 fmdmtal;    /* FM DMA transfer bus address low reg 0x18 */
 437         u32 fmdmtcid;   /* FM DMA transfer bus communication ID reg 0x1c */
 438         u32 fmdmra;     /* FM DMA bus internal ram address register 0x20 */
 439         u32 fmdmrd;     /* FM DMA bus internal ram data register 0x24 */
 440         u32 fmdmwcr;    /* FM DMA CAM watchdog counter value 0x28 */
 441         u32 fmdmebcr;   /* FM DMA CAM base in MURAM register 0x2c */
 442         u32 fmdmccqdr;  /* FM DMA CAM and CMD Queue Debug reg 0x30 */
 443         u32 fmdmccqvr1; /* FM DMA CAM and CMD Queue Value reg #1 0x34 */
 444         u32 fmdmccqvr2; /* FM DMA CAM and CMD Queue Value reg #2 0x38 */
 445         u32 fmdmcqvr3;  /* FM DMA CMD Queue Value register #3 0x3c */
 446         u32 fmdmcqvr4;  /* FM DMA CMD Queue Value register #4 0x40 */
 447         u32 fmdmcqvr5;  /* FM DMA CMD Queue Value register #5 0x44 */
 448         u32 fmdmsefrc;  /* FM DMA Semaphore Entry Full Reject Cntr 0x48 */
 449         u32 fmdmsqfrc;  /* FM DMA Semaphore Queue Full Reject Cntr 0x4c */
 450         u32 fmdmssrc;   /* FM DMA Semaphore SYNC Reject Counter 0x50 */
 451         u32 fmdmdcr;    /* FM DMA Debug Counter 0x54 */
 452         u32 fmdmemsr;   /* FM DMA Emergency Smoother Register 0x58 */
 453         u32 res005c;    /* 0x5c */
 454         u32 fmdmplr[FMAN_LIODN_TBL / 2];        /* DMA LIODN regs 0x60-0xdf */
 455         u32 res00e0[0x400 - 56];
 456 };
 457
 458 struct fman_hwp_regs {
 459         u32 res0000[0x844 / 4];         /* 0x000..0x843 */
 460         u32 fmprrpimac; /* FM Parser Internal memory access control */
 461         u32 res[(0x1000 - 0x848) / 4];  /* 0x848..0xFFF */
 462 };
 463
 464 /* Structure that holds current FMan state.
 465  * Used for saving run time information.
 466  */
 467 struct fman_state_struct {
 468         u8 fm_id;
 469         u16 fm_clk_freq;
 470         struct fman_rev_info rev_info;
 471         bool enabled_time_stamp;
 472         u8 count1_micro_bit;
 473         u8 total_num_of_tasks;
 474         u8 accumulated_num_of_tasks;
 475         u32 accumulated_fifo_size;
 476         u8 accumulated_num_of_open_dmas;
 477         u8 accumulated_num_of_deq_tnums;
 478         u32 exceptions;
 479         u32 extra_fifo_pool_size;
 480         u8 extra_tasks_pool_size;
 481         u8 extra_open_dmas_pool_size;
 482         u16 port_mfl[MAX_NUM_OF_MACS];
 483         u16 mac_mfl[MAX_NUM_OF_MACS];
 484
 485         /* SOC specific */
 486         u32 fm_iram_size;
 487         /* DMA */
 488         u32 dma_thresh_max_commq;
 489         u32 dma_thresh_max_buf;
 490         u32 max_num_of_open_dmas;
 491         /* QMI */
 492         u32 qmi_max_num_of_tnums;
 493         u32 qmi_def_tnums_thresh;
 494         /* BMI */
 495         u32 bmi_max_num_of_tasks;
 496         u32 bmi_max_fifo_size;
 497         /* General */
 498         u32 fm_port_num_of_cg;
 499         u32 num_of_rx_ports;
 500         u32 total_fifo_size;
 501
 502         u32 qman_channel_base;
 503         u32 num_of_qman_channels;
 504
 505         struct resource *res;
 506 };
 507
 508 /* Structure that holds FMan initial configuration */
 509 struct fman_cfg {
 510         u8 disp_limit_tsh;
 511         u8 prs_disp_tsh;
 512         u8 plcr_disp_tsh;
 513         u8 kg_disp_tsh;
 514         u8 bmi_disp_tsh;
 515         u8 qmi_enq_disp_tsh;
 516         u8 qmi_deq_disp_tsh;
 517         u8 fm_ctl1_disp_tsh;
 518         u8 fm_ctl2_disp_tsh;
 519         int dma_cache_override;
 520         enum fman_dma_aid_mode dma_aid_mode;
 521         u32 dma_axi_dbg_num_of_beats;
 522         u32 dma_cam_num_of_entries;
 523         u32 dma_watchdog;
 524         u8 dma_comm_qtsh_asrt_emer;
 525         u32 dma_write_buf_tsh_asrt_emer;
 526         u32 dma_read_buf_tsh_asrt_emer;
 527         u8 dma_comm_qtsh_clr_emer;
 528         u32 dma_write_buf_tsh_clr_emer;
 529         u32 dma_read_buf_tsh_clr_emer;
 530         u32 dma_sos_emergency;
 531         int dma_dbg_cnt_mode;
 532         int catastrophic_err;
 533         int dma_err;
 534         u32 exceptions;
 535         u16 clk_freq;
 536         u32 cam_base_addr;
 537         u32 fifo_base_addr;
 538         u32 total_fifo_size;
 539         u32 total_num_of_tasks;
 540         u32 qmi_def_tnums_thresh;
 541 };
 542
 543 #ifdef CONFIG_DPAA_ERRATUM_A050385
 544 static bool fman_has_err_a050385;
 545 #endif
 546
 547 static irqreturn_t fman_exceptions(struct fman *fman,
 548                                    enum fman_exceptions exception)
 549 {
 550         dev_dbg(fman->dev, "%s: FMan[%d] exception %d\n",
 551                 __func__, fman->state->fm_id, exception);
 552
 553         return IRQ_HANDLED;
 554 }
 555
 556 static irqreturn_t fman_bus_error(struct fman *fman, u8 __maybe_unused port_id,
 557                                   u64 __maybe_unused addr,
 558                                   u8 __maybe_unused tnum,
 559                                   u16 __maybe_unused liodn)
 560 {
 561         dev_dbg(fman->dev, "%s: FMan[%d] bus error: port_id[%d]\n",
 562                 __func__, fman->state->fm_id, port_id);
 563
 564         return IRQ_HANDLED;
 565 }
 566
 567 static inline irqreturn_t call_mac_isr(struct fman *fman, u8 id)
 568 {
 569         if (fman->intr_mng[id].isr_cb) {
 570                 fman->intr_mng[id].isr_cb(fman->intr_mng[id].src_handle);
 571
 572                 return IRQ_HANDLED;
 573         }
 574
 575         return IRQ_NONE;
 576 }
 577
 578 static inline u8 hw_port_id_to_sw_port_id(u8 major, u8 hw_port_id)
 579 {
 580         u8 sw_port_id = 0;
 581
 582         if (hw_port_id >= BASE_TX_PORTID)
 583                 sw_port_id = hw_port_id - BASE_TX_PORTID;
 584         else if (hw_port_id >= BASE_RX_PORTID)
 585                 sw_port_id = hw_port_id - BASE_RX_PORTID;
 586         else
 587                 sw_port_id = 0;
 588
 589         return sw_port_id;
 590 }
 591
 592 static void set_port_order_restoration(struct fman_fpm_regs __iomem *fpm_rg,
 593                                        u8 port_id)
 594 {
 595         u32 tmp = 0;
 596
 597         tmp = port_id << FPM_PORT_FM_CTL_PORTID_SHIFT;
 598
 599         tmp |= FPM_PRT_FM_CTL2 | FPM_PRT_FM_CTL1;
 600
 601         /* order restoration */
 602         if (port_id % 2)
 603                 tmp |= FPM_PRT_FM_CTL1 << FPM_PRC_ORA_FM_CTL_SEL_SHIFT;
 604         else
 605                 tmp |= FPM_PRT_FM_CTL2 << FPM_PRC_ORA_FM_CTL_SEL_SHIFT;
 606
 607         iowrite32be(tmp, &fpm_rg->fmfp_prc);
 608 }
 609
 610 static void set_port_liodn(struct fman *fman, u8 port_id,
 611                            u32 liodn_base, u32 liodn_ofst)
 612 {
 613         u32 tmp;
 614
 615         iowrite32be(liodn_ofst, &fman->bmi_regs->fmbm_spliodn[port_id - 1]);
 616         if (!IS_ENABLED(CONFIG_FSL_PAMU))
 617                 return;
 618         /* set LIODN base for this port */
 619         tmp = ioread32be(&fman->dma_regs->fmdmplr[port_id / 2]);
 620         if (port_id % 2) {
 621                 tmp &= ~DMA_LIODN_BASE_MASK;
 622                 tmp |= liodn_base;
 623         } else {
 624                 tmp &= ~(DMA_LIODN_BASE_MASK << DMA_LIODN_SHIFT);
 625                 tmp |= liodn_base << DMA_LIODN_SHIFT;
 626         }
 627         iowrite32be(tmp, &fman->dma_regs->fmdmplr[port_id / 2]);
 628 }
 629
 630 static void enable_rams_ecc(struct fman_fpm_regs __iomem *fpm_rg)
 631 {
 632         u32 tmp;
 633
 634         tmp = ioread32be(&fpm_rg->fm_rcr);
 635         if (tmp & FPM_RAM_RAMS_ECC_EN_SRC_SEL)
 636                 iowrite32be(tmp | FPM_RAM_IRAM_ECC_EN, &fpm_rg->fm_rcr);
 637         else
 638                 iowrite32be(tmp | FPM_RAM_RAMS_ECC_EN |
 639                             FPM_RAM_IRAM_ECC_EN, &fpm_rg->fm_rcr);
 640 }
 641
 642 static void disable_rams_ecc(struct fman_fpm_regs __iomem *fpm_rg)
 643 {
 644         u32 tmp;
 645
 646         tmp = ioread32be(&fpm_rg->fm_rcr);
 647         if (tmp & FPM_RAM_RAMS_ECC_EN_SRC_SEL)
 648                 iowrite32be(tmp & ~FPM_RAM_IRAM_ECC_EN, &fpm_rg->fm_rcr);
 649         else
 650                 iowrite32be(tmp & ~(FPM_RAM_RAMS_ECC_EN | FPM_RAM_IRAM_ECC_EN),
 651                             &fpm_rg->fm_rcr);
 652 }
 653
 654 static void fman_defconfig(struct fman_cfg *cfg)
 655 {
 656         memset(cfg, 0, sizeof(struct fman_cfg));
 657
 658         cfg->catastrophic_err = DEFAULT_CATASTROPHIC_ERR;
 659         cfg->dma_err = DEFAULT_DMA_ERR;
 660         cfg->dma_aid_mode = DEFAULT_AID_MODE;
 661         cfg->dma_comm_qtsh_clr_emer = DEFAULT_DMA_COMM_Q_LOW;
 662         cfg->dma_comm_qtsh_asrt_emer = DEFAULT_DMA_COMM_Q_HIGH;
 663         cfg->dma_cache_override = DEFAULT_CACHE_OVERRIDE;
 664         cfg->dma_cam_num_of_entries = DEFAULT_DMA_CAM_NUM_OF_ENTRIES;
 665         cfg->dma_dbg_cnt_mode = DEFAULT_DMA_DBG_CNT_MODE;
 666         cfg->dma_sos_emergency = DEFAULT_DMA_SOS_EMERGENCY;
 667         cfg->dma_watchdog = DEFAULT_DMA_WATCHDOG;
 668         cfg->disp_limit_tsh = DEFAULT_DISP_LIMIT;
 669         cfg->prs_disp_tsh = DEFAULT_PRS_DISP_TH;
 670         cfg->plcr_disp_tsh = DEFAULT_PLCR_DISP_TH;
 671         cfg->kg_disp_tsh = DEFAULT_KG_DISP_TH;
 672         cfg->bmi_disp_tsh = DEFAULT_BMI_DISP_TH;
 673         cfg->qmi_enq_disp_tsh = DEFAULT_QMI_ENQ_DISP_TH;
 674         cfg->qmi_deq_disp_tsh = DEFAULT_QMI_DEQ_DISP_TH;
 675         cfg->fm_ctl1_disp_tsh = DEFAULT_FM_CTL1_DISP_TH;
 676         cfg->fm_ctl2_disp_tsh = DEFAULT_FM_CTL2_DISP_TH;
 677 }
 678
 679 static int dma_init(struct fman *fman)
 680 {
 681         struct fman_dma_regs __iomem *dma_rg = fman->dma_regs;
 682         struct fman_cfg *cfg = fman->cfg;
 683         u32 tmp_reg;
 684
 685         /* Init DMA Registers */
 686
 687         /* clear status reg events */
 688         tmp_reg = (DMA_STATUS_BUS_ERR | DMA_STATUS_READ_ECC |
 689                    DMA_STATUS_SYSTEM_WRITE_ECC | DMA_STATUS_FM_WRITE_ECC);
 690         iowrite32be(ioread32be(&dma_rg->fmdmsr) | tmp_reg, &dma_rg->fmdmsr);
 691
 692         /* configure mode register */
 693         tmp_reg = 0;
 694         tmp_reg |= cfg->dma_cache_override << DMA_MODE_CACHE_OR_SHIFT;
 695         if (cfg->exceptions & EX_DMA_BUS_ERROR)
 696                 tmp_reg |= DMA_MODE_BER;
 697         if ((cfg->exceptions & EX_DMA_SYSTEM_WRITE_ECC) |
 698             (cfg->exceptions & EX_DMA_READ_ECC) |
 699             (cfg->exceptions & EX_DMA_FM_WRITE_ECC))
 700                 tmp_reg |= DMA_MODE_ECC;
 701         if (cfg->dma_axi_dbg_num_of_beats)
 702                 tmp_reg |= (DMA_MODE_AXI_DBG_MASK &
 703                         ((cfg->dma_axi_dbg_num_of_beats - 1)
 704                         << DMA_MODE_AXI_DBG_SHIFT));
 705
 706         tmp_reg |= (((cfg->dma_cam_num_of_entries / DMA_CAM_UNITS) - 1) &
 707                 DMA_MODE_CEN_MASK) << DMA_MODE_CEN_SHIFT;
 708         tmp_reg |= DMA_MODE_SECURE_PROT;
 709         tmp_reg |= cfg->dma_dbg_cnt_mode << DMA_MODE_DBG_SHIFT;
 710         tmp_reg |= cfg->dma_aid_mode << DMA_MODE_AID_MODE_SHIFT;
 711
 712         iowrite32be(tmp_reg, &dma_rg->fmdmmr);
 713
 714         /* configure thresholds register */
 715         tmp_reg = ((u32)cfg->dma_comm_qtsh_asrt_emer <<
 716                 DMA_THRESH_COMMQ_SHIFT);
 717         tmp_reg |= (cfg->dma_read_buf_tsh_asrt_emer &
 718                 DMA_THRESH_READ_INT_BUF_MASK) << DMA_THRESH_READ_INT_BUF_SHIFT;
 719         tmp_reg |= cfg->dma_write_buf_tsh_asrt_emer &
 720                 DMA_THRESH_WRITE_INT_BUF_MASK;
 721
 722         iowrite32be(tmp_reg, &dma_rg->fmdmtr);
 723
 724         /* configure hysteresis register */
 725         tmp_reg = ((u32)cfg->dma_comm_qtsh_clr_emer <<
 726                 DMA_THRESH_COMMQ_SHIFT);
 727         tmp_reg |= (cfg->dma_read_buf_tsh_clr_emer &
 728                 DMA_THRESH_READ_INT_BUF_MASK) << DMA_THRESH_READ_INT_BUF_SHIFT;
 729         tmp_reg |= cfg->dma_write_buf_tsh_clr_emer &
 730                 DMA_THRESH_WRITE_INT_BUF_MASK;
 731
 732         iowrite32be(tmp_reg, &dma_rg->fmdmhy);
 733
 734         /* configure emergency threshold */
 735         iowrite32be(cfg->dma_sos_emergency, &dma_rg->fmdmsetr);
 736
 737         /* configure Watchdog */
 738         iowrite32be((cfg->dma_watchdog * cfg->clk_freq), &dma_rg->fmdmwcr);
 739
 740         iowrite32be(cfg->cam_base_addr, &dma_rg->fmdmebcr);
 741
 742         /* Allocate MURAM for CAM */
 743         fman->cam_size =
 744                 (u32)(fman->cfg->dma_cam_num_of_entries * DMA_CAM_SIZEOF_ENTRY);
 745         fman->cam_offset = fman_muram_alloc(fman->muram, fman->cam_size);
 746         if (IS_ERR_VALUE(fman->cam_offset)) {
 747                 dev_err(fman->dev, "%s: MURAM alloc for DMA CAM failed\n",
 748                         __func__);
 749                 return -ENOMEM;
 750         }
 751
 752         if (fman->state->rev_info.major == 2) {
 753                 u32 __iomem *cam_base_addr;
 754
 755                 fman_muram_free_mem(fman->muram, fman->cam_offset,
 756                                     fman->cam_size);
 757
 758                 fman->cam_size = fman->cfg->dma_cam_num_of_entries * 72 + 128;
 759                 fman->cam_offset = fman_muram_alloc(fman->muram,
 760                                                     fman->cam_size);
 761                 if (IS_ERR_VALUE(fman->cam_offset)) {
 762                         dev_err(fman->dev, "%s: MURAM alloc for DMA CAM failed\n",
 763                                 __func__);
 764                         return -ENOMEM;
 765                 }
 766
 767                 if (fman->cfg->dma_cam_num_of_entries % 8 ||
 768                     fman->cfg->dma_cam_num_of_entries > 32) {
 769                         dev_err(fman->dev, "%s: wrong dma_cam_num_of_entries\n",
 770                                 __func__);
 771                         return -EINVAL;
 772                 }
 773
 774                 cam_base_addr = (u32 __iomem *)
 775                         fman_muram_offset_to_vbase(fman->muram,
 776                                                    fman->cam_offset);
 777                 iowrite32be(~((1 <<
 778                             (32 - fman->cfg->dma_cam_num_of_entries)) - 1),
 779                             cam_base_addr);
 780         }
 781
 782         fman->cfg->cam_base_addr = fman->cam_offset;
 783
 784         return 0;
 785 }
 786
 787 static void fpm_init(struct fman_fpm_regs __iomem *fpm_rg, struct fman_cfg *cfg)
 788 {
 789         u32 tmp_reg;
 790         int i;
 791
 792         /* Init FPM Registers */
 793
 794         tmp_reg = (u32)(cfg->disp_limit_tsh << FPM_DISP_LIMIT_SHIFT);
 795         iowrite32be(tmp_reg, &fpm_rg->fmfp_mxd);
 796
 797         tmp_reg = (((u32)cfg->prs_disp_tsh << FPM_THR1_PRS_SHIFT) |
 798                    ((u32)cfg->kg_disp_tsh << FPM_THR1_KG_SHIFT) |
 799                    ((u32)cfg->plcr_disp_tsh << FPM_THR1_PLCR_SHIFT) |
 800                    ((u32)cfg->bmi_disp_tsh << FPM_THR1_BMI_SHIFT));
 801         iowrite32be(tmp_reg, &fpm_rg->fmfp_dist1);
 802
 803         tmp_reg =
 804                 (((u32)cfg->qmi_enq_disp_tsh << FPM_THR2_QMI_ENQ_SHIFT) |
 805                  ((u32)cfg->qmi_deq_disp_tsh << FPM_THR2_QMI_DEQ_SHIFT) |
 806                  ((u32)cfg->fm_ctl1_disp_tsh << FPM_THR2_FM_CTL1_SHIFT) |
 807                  ((u32)cfg->fm_ctl2_disp_tsh << FPM_THR2_FM_CTL2_SHIFT));
 808         iowrite32be(tmp_reg, &fpm_rg->fmfp_dist2);
 809
 810         /* define exceptions and error behavior */
 811         tmp_reg = 0;
 812         /* Clear events */
 813         tmp_reg |= (FPM_EV_MASK_STALL | FPM_EV_MASK_DOUBLE_ECC |
 814                     FPM_EV_MASK_SINGLE_ECC);
 815         /* enable interrupts */
 816         if (cfg->exceptions & EX_FPM_STALL_ON_TASKS)
 817                 tmp_reg |= FPM_EV_MASK_STALL_EN;
 818         if (cfg->exceptions & EX_FPM_SINGLE_ECC)
 819                 tmp_reg |= FPM_EV_MASK_SINGLE_ECC_EN;
 820         if (cfg->exceptions & EX_FPM_DOUBLE_ECC)
 821                 tmp_reg |= FPM_EV_MASK_DOUBLE_ECC_EN;
 822         tmp_reg |= (cfg->catastrophic_err << FPM_EV_MASK_CAT_ERR_SHIFT);
 823         tmp_reg |= (cfg->dma_err << FPM_EV_MASK_DMA_ERR_SHIFT);
 824         /* FMan is not halted upon external halt activation */
 825         tmp_reg |= FPM_EV_MASK_EXTERNAL_HALT;
 826         /* Man is not halted upon  Unrecoverable ECC error behavior */
 827         tmp_reg |= FPM_EV_MASK_ECC_ERR_HALT;
 828         iowrite32be(tmp_reg, &fpm_rg->fmfp_ee);
 829
 830         /* clear all fmCtls event registers */
 831         for (i = 0; i < FM_NUM_OF_FMAN_CTRL_EVENT_REGS; i++)
 832                 iowrite32be(0xFFFFFFFF, &fpm_rg->fmfp_cev[i]);
 833
 834         /* RAM ECC -  enable and clear events */
 835         /* first we need to clear all parser memory,
 836          * as it is uninitialized and may cause ECC errors
 837          */
 838         /* event bits */
 839         tmp_reg = (FPM_RAM_MURAM_ECC | FPM_RAM_IRAM_ECC);
 840
 841         iowrite32be(tmp_reg, &fpm_rg->fm_rcr);
 842
 843         tmp_reg = 0;
 844         if (cfg->exceptions & EX_IRAM_ECC) {
 845                 tmp_reg |= FPM_IRAM_ECC_ERR_EX_EN;
 846                 enable_rams_ecc(fpm_rg);
 847         }
 848         if (cfg->exceptions & EX_MURAM_ECC) {
 849                 tmp_reg |= FPM_MURAM_ECC_ERR_EX_EN;
 850                 enable_rams_ecc(fpm_rg);
 851         }
 852         iowrite32be(tmp_reg, &fpm_rg->fm_rie);
 853 }
 854
 855 static void bmi_init(struct fman_bmi_regs __iomem *bmi_rg,
 856                      struct fman_cfg *cfg)
 857 {
 858         u32 tmp_reg;
 859
 860         /* Init BMI Registers */
 861
 862         /* define common resources */
 863         tmp_reg = cfg->fifo_base_addr;
 864         tmp_reg = tmp_reg / BMI_FIFO_ALIGN;
 865
 866         tmp_reg |= ((cfg->total_fifo_size / FMAN_BMI_FIFO_UNITS - 1) <<
 867                     BMI_CFG1_FIFO_SIZE_SHIFT);
 868         iowrite32be(tmp_reg, &bmi_rg->fmbm_cfg1);
 869
 870         tmp_reg = ((cfg->total_num_of_tasks - 1) & BMI_CFG2_TASKS_MASK) <<
 871                    BMI_CFG2_TASKS_SHIFT;
 872         /* num of DMA's will be dynamically updated when each port is set */
 873         iowrite32be(tmp_reg, &bmi_rg->fmbm_cfg2);
 874
 875         /* define unmaskable exceptions, enable and clear events */
 876         tmp_reg = 0;
 877         iowrite32be(BMI_ERR_INTR_EN_LIST_RAM_ECC |
 878                     BMI_ERR_INTR_EN_STORAGE_PROFILE_ECC |
 879                     BMI_ERR_INTR_EN_STATISTICS_RAM_ECC |
 880                     BMI_ERR_INTR_EN_DISPATCH_RAM_ECC, &bmi_rg->fmbm_ievr);
 881
 882         if (cfg->exceptions & EX_BMI_LIST_RAM_ECC)
 883                 tmp_reg |= BMI_ERR_INTR_EN_LIST_RAM_ECC;
 884         if (cfg->exceptions & EX_BMI_STORAGE_PROFILE_ECC)
 885                 tmp_reg |= BMI_ERR_INTR_EN_STORAGE_PROFILE_ECC;
 886         if (cfg->exceptions & EX_BMI_STATISTICS_RAM_ECC)
 887                 tmp_reg |= BMI_ERR_INTR_EN_STATISTICS_RAM_ECC;
 888         if (cfg->exceptions & EX_BMI_DISPATCH_RAM_ECC)
 889                 tmp_reg |= BMI_ERR_INTR_EN_DISPATCH_RAM_ECC;
 890         iowrite32be(tmp_reg, &bmi_rg->fmbm_ier);
 891 }
 892
 893 static void qmi_init(struct fman_qmi_regs __iomem *qmi_rg,
 894                      struct fman_cfg *cfg)
 895 {
 896         u32 tmp_reg;
 897
 898         /* Init QMI Registers */
 899
 900         /* Clear error interrupt events */
 901
 902         iowrite32be(QMI_ERR_INTR_EN_DOUBLE_ECC | QMI_ERR_INTR_EN_DEQ_FROM_DEF,
 903                     &qmi_rg->fmqm_eie);
 904         tmp_reg = 0;
 905         if (cfg->exceptions & EX_QMI_DEQ_FROM_UNKNOWN_PORTID)
 906                 tmp_reg |= QMI_ERR_INTR_EN_DEQ_FROM_DEF;
 907         if (cfg->exceptions & EX_QMI_DOUBLE_ECC)
 908                 tmp_reg |= QMI_ERR_INTR_EN_DOUBLE_ECC;
 909         /* enable events */
 910         iowrite32be(tmp_reg, &qmi_rg->fmqm_eien);
 911
 912         tmp_reg = 0;
 913         /* Clear interrupt events */
 914         iowrite32be(QMI_INTR_EN_SINGLE_ECC, &qmi_rg->fmqm_ie);
 915         if (cfg->exceptions & EX_QMI_SINGLE_ECC)
 916                 tmp_reg |= QMI_INTR_EN_SINGLE_ECC;
 917         /* enable events */
 918         iowrite32be(tmp_reg, &qmi_rg->fmqm_ien);
 919 }
 920
 921 static void hwp_init(struct fman_hwp_regs __iomem *hwp_rg)
 922 {
 923         /* enable HW Parser */
 924         iowrite32be(HWP_RPIMAC_PEN, &hwp_rg->fmprrpimac);
 925 }
 926
 927 static int enable(struct fman *fman, struct fman_cfg *cfg)
 928 {
 929         u32 cfg_reg = 0;
 930
 931         /* Enable all modules */
 932
 933         /* clear&enable global counters - calculate reg and save for later,
 934          * because it's the same reg for QMI enable
 935          */
 936         cfg_reg = QMI_CFG_EN_COUNTERS;
 937
 938         /* Set enqueue and dequeue thresholds */
 939         cfg_reg |= (cfg->qmi_def_tnums_thresh << 8) | cfg->qmi_def_tnums_thresh;
 940
 941         iowrite32be(BMI_INIT_START, &fman->bmi_regs->fmbm_init);
 942         iowrite32be(cfg_reg | QMI_CFG_ENQ_EN | QMI_CFG_DEQ_EN,
 943                     &fman->qmi_regs->fmqm_gc);
 944
 945         return 0;
 946 }
 947
 948 static int set_exception(struct fman *fman,
 949                          enum fman_exceptions exception, bool enable)
 950 {
 951         u32 tmp;
 952
 953         switch (exception) {
 954         case FMAN_EX_DMA_BUS_ERROR:
 955                 tmp = ioread32be(&fman->dma_regs->fmdmmr);
 956                 if (enable)
 957                         tmp |= DMA_MODE_BER;
 958                 else
 959                         tmp &= ~DMA_MODE_BER;
 960                 /* disable bus error */
 961                 iowrite32be(tmp, &fman->dma_regs->fmdmmr);
 962                 break;
 963         case FMAN_EX_DMA_READ_ECC:
 964         case FMAN_EX_DMA_SYSTEM_WRITE_ECC:
 965         case FMAN_EX_DMA_FM_WRITE_ECC:
 966                 tmp = ioread32be(&fman->dma_regs->fmdmmr);
 967                 if (enable)
 968                         tmp |= DMA_MODE_ECC;
 969                 else
 970                         tmp &= ~DMA_MODE_ECC;
 971                 iowrite32be(tmp, &fman->dma_regs->fmdmmr);
 972                 break;
 973         case FMAN_EX_FPM_STALL_ON_TASKS:
 974                 tmp = ioread32be(&fman->fpm_regs->fmfp_ee);
 975                 if (enable)
 976                         tmp |= FPM_EV_MASK_STALL_EN;
 977                 else
 978                         tmp &= ~FPM_EV_MASK_STALL_EN;
 979                 iowrite32be(tmp, &fman->fpm_regs->fmfp_ee);
 980                 break;
 981         case FMAN_EX_FPM_SINGLE_ECC:
 982                 tmp = ioread32be(&fman->fpm_regs->fmfp_ee);
 983                 if (enable)
 984                         tmp |= FPM_EV_MASK_SINGLE_ECC_EN;
 985                 else
 986                         tmp &= ~FPM_EV_MASK_SINGLE_ECC_EN;
 987                 iowrite32be(tmp, &fman->fpm_regs->fmfp_ee);
 988                 break;
 989         case FMAN_EX_FPM_DOUBLE_ECC:
 990                 tmp = ioread32be(&fman->fpm_regs->fmfp_ee);
 991                 if (enable)
 992                         tmp |= FPM_EV_MASK_DOUBLE_ECC_EN;
 993                 else
 994                         tmp &= ~FPM_EV_MASK_DOUBLE_ECC_EN;
 995                 iowrite32be(tmp, &fman->fpm_regs->fmfp_ee);
 996                 break;
 997         case FMAN_EX_QMI_SINGLE_ECC:
 998                 tmp = ioread32be(&fman->qmi_regs->fmqm_ien);
 999                 if (enable)
1000                         tmp |= QMI_INTR_EN_SINGLE_ECC;
1001                 else
1002                         tmp &= ~QMI_INTR_EN_SINGLE_ECC;
1003                 iowrite32be(tmp, &fman->qmi_regs->fmqm_ien);
1004                 break;
1005         case FMAN_EX_QMI_DOUBLE_ECC:
1006                 tmp = ioread32be(&fman->qmi_regs->fmqm_eien);
1007                 if (enable)
1008                         tmp |= QMI_ERR_INTR_EN_DOUBLE_ECC;
1009                 else
1010                         tmp &= ~QMI_ERR_INTR_EN_DOUBLE_ECC;
1011                 iowrite32be(tmp, &fman->qmi_regs->fmqm_eien);
1012                 break;
1013         case FMAN_EX_QMI_DEQ_FROM_UNKNOWN_PORTID:
1014                 tmp = ioread32be(&fman->qmi_regs->fmqm_eien);
1015                 if (enable)
1016                         tmp |= QMI_ERR_INTR_EN_DEQ_FROM_DEF;
1017                 else
1018                         tmp &= ~QMI_ERR_INTR_EN_DEQ_FROM_DEF;
1019                 iowrite32be(tmp, &fman->qmi_regs->fmqm_eien);
1020                 break;
1021         case FMAN_EX_BMI_LIST_RAM_ECC:
1022                 tmp = ioread32be(&fman->bmi_regs->fmbm_ier);
1023                 if (enable)
1024                         tmp |= BMI_ERR_INTR_EN_LIST_RAM_ECC;
1025                 else
1026                         tmp &= ~BMI_ERR_INTR_EN_LIST_RAM_ECC;
1027                 iowrite32be(tmp, &fman->bmi_regs->fmbm_ier);
1028                 break;
1029         case FMAN_EX_BMI_STORAGE_PROFILE_ECC:
1030                 tmp = ioread32be(&fman->bmi_regs->fmbm_ier);
1031                 if (enable)
1032                         tmp |= BMI_ERR_INTR_EN_STORAGE_PROFILE_ECC;
1033                 else
1034                         tmp &= ~BMI_ERR_INTR_EN_STORAGE_PROFILE_ECC;
1035                 iowrite32be(tmp, &fman->bmi_regs->fmbm_ier);
1036                 break;
1037         case FMAN_EX_BMI_STATISTICS_RAM_ECC:
1038                 tmp = ioread32be(&fman->bmi_regs->fmbm_ier);
1039                 if (enable)
1040                         tmp |= BMI_ERR_INTR_EN_STATISTICS_RAM_ECC;
1041                 else
1042                         tmp &= ~BMI_ERR_INTR_EN_STATISTICS_RAM_ECC;
1043                 iowrite32be(tmp, &fman->bmi_regs->fmbm_ier);
1044                 break;
1045         case FMAN_EX_BMI_DISPATCH_RAM_ECC:
1046                 tmp = ioread32be(&fman->bmi_regs->fmbm_ier);
1047                 if (enable)
1048                         tmp |= BMI_ERR_INTR_EN_DISPATCH_RAM_ECC;
1049                 else
1050                         tmp &= ~BMI_ERR_INTR_EN_DISPATCH_RAM_ECC;
1051                 iowrite32be(tmp, &fman->bmi_regs->fmbm_ier);
1052                 break;
1053         case FMAN_EX_IRAM_ECC:
1054                 tmp = ioread32be(&fman->fpm_regs->fm_rie);
1055                 if (enable) {
1056                         /* enable ECC if not enabled */
1057                         enable_rams_ecc(fman->fpm_regs);
1058                         /* enable ECC interrupts */
1059                         tmp |= FPM_IRAM_ECC_ERR_EX_EN;
1060                 } else {
1061                         /* ECC mechanism may be disabled,
1062                          * depending on driver status
1063                          */
1064                         disable_rams_ecc(fman->fpm_regs);
1065                         tmp &= ~FPM_IRAM_ECC_ERR_EX_EN;
1066                 }
1067                 iowrite32be(tmp, &fman->fpm_regs->fm_rie);
1068                 break;
1069         case FMAN_EX_MURAM_ECC:
1070                 tmp = ioread32be(&fman->fpm_regs->fm_rie);
1071                 if (enable) {
1072                         /* enable ECC if not enabled */
1073                         enable_rams_ecc(fman->fpm_regs);
1074                         /* enable ECC interrupts */
1075                         tmp |= FPM_MURAM_ECC_ERR_EX_EN;
1076                 } else {
1077                         /* ECC mechanism may be disabled,
1078                          * depending on driver status
1079                          */
1080                         disable_rams_ecc(fman->fpm_regs);
1081                         tmp &= ~FPM_MURAM_ECC_ERR_EX_EN;
1082                 }
1083                 iowrite32be(tmp, &fman->fpm_regs->fm_rie);
1084                 break;
1085         default:
1086                 return -EINVAL;
1087         }
1088         return 0;
1089 }
1090
1091 static void resume(struct fman_fpm_regs __iomem *fpm_rg)
1092 {
1093         u32 tmp;
1094
1095         tmp = ioread32be(&fpm_rg->fmfp_ee);
1096         /* clear tmp_reg event bits in order not to clear standing events */
1097         tmp &= ~(FPM_EV_MASK_DOUBLE_ECC |
1098                  FPM_EV_MASK_STALL | FPM_EV_MASK_SINGLE_ECC);
1099         tmp |= FPM_EV_MASK_RELEASE_FM;
1100
1101         iowrite32be(tmp, &fpm_rg->fmfp_ee);
1102 }
1103
1104 static int fill_soc_specific_params(struct fman_state_struct *state)
1105 {
1106         u8 minor = state->rev_info.minor;
1107         /* P4080 - Major 2
1108          * P2041/P3041/P5020/P5040 - Major 3
1109          * Tx/Bx - Major 6
1110          */
1111         switch (state->rev_info.major) {
1112         case 3:
1113                 state->bmi_max_fifo_size        = 160 * 1024;
1114                 state->fm_iram_size             = 64 * 1024;
1115                 state->dma_thresh_max_commq     = 31;
1116                 state->dma_thresh_max_buf       = 127;
1117                 state->qmi_max_num_of_tnums     = 64;
1118                 state->qmi_def_tnums_thresh     = 48;
1119                 state->bmi_max_num_of_tasks     = 128;
1120                 state->max_num_of_open_dmas     = 32;
1121                 state->fm_port_num_of_cg        = 256;
1122                 state->num_of_rx_ports  = 6;
1123                 state->total_fifo_size  = 136 * 1024;
1124                 break;
1125
1126         case 2:
1127                 state->bmi_max_fifo_size        = 160 * 1024;
1128                 state->fm_iram_size             = 64 * 1024;
1129                 state->dma_thresh_max_commq     = 31;
1130                 state->dma_thresh_max_buf       = 127;
1131                 state->qmi_max_num_of_tnums     = 64;
1132                 state->qmi_def_tnums_thresh     = 48;
1133                 state->bmi_max_num_of_tasks     = 128;
1134                 state->max_num_of_open_dmas     = 32;
1135                 state->fm_port_num_of_cg        = 256;
1136                 state->num_of_rx_ports  = 5;
1137                 state->total_fifo_size  = 100 * 1024;
1138                 break;
1139
1140         case 6:
1141                 state->dma_thresh_max_commq     = 83;
1142                 state->dma_thresh_max_buf       = 127;
1143                 state->qmi_max_num_of_tnums     = 64;
1144                 state->qmi_def_tnums_thresh     = 32;
1145                 state->fm_port_num_of_cg        = 256;
1146
1147                 /* FManV3L */
1148                 if (minor == 1 || minor == 4) {
1149                         state->bmi_max_fifo_size        = 192 * 1024;
1150                         state->bmi_max_num_of_tasks     = 64;
1151                         state->max_num_of_open_dmas     = 32;
1152                         state->num_of_rx_ports          = 5;
1153                         if (minor == 1)
1154                                 state->fm_iram_size     = 32 * 1024;
1155                         else
1156                                 state->fm_iram_size     = 64 * 1024;
1157                         state->total_fifo_size          = 156 * 1024;
1158                 }
1159                 /* FManV3H */
1160                 else if (minor == 0 || minor == 2 || minor == 3) {
1161                         state->bmi_max_fifo_size        = 384 * 1024;
1162                         state->fm_iram_size             = 64 * 1024;
1163                         state->bmi_max_num_of_tasks     = 128;
1164                         state->max_num_of_open_dmas     = 84;
1165                         state->num_of_rx_ports          = 8;
1166                         state->total_fifo_size          = 295 * 1024;
1167                 } else {
1168                         pr_err("Unsupported FManv3 version\n");
1169                         return -EINVAL;
1170                 }
1171
1172                 break;
1173         default:
1174                 pr_err("Unsupported FMan version\n");
1175                 return -EINVAL;
1176         }
1177
1178         return 0;
1179 }
1180
1181 static bool is_init_done(struct fman_cfg *cfg)
1182 {
1183         /* Checks if FMan driver parameters were initialized */
1184         if (!cfg)
1185                 return true;
1186
1187         return false;
1188 }
1189
1190 static void free_init_resources(struct fman *fman)
1191 {
1192         if (fman->cam_offset)
1193                 fman_muram_free_mem(fman->muram, fman->cam_offset,
1194                                     fman->cam_size);
1195         if (fman->fifo_offset)
1196                 fman_muram_free_mem(fman->muram, fman->fifo_offset,
1197                                     fman->fifo_size);
1198 }
1199
1200 static irqreturn_t bmi_err_event(struct fman *fman)
1201 {
1202         u32 event, mask, force;
1203         struct fman_bmi_regs __iomem *bmi_rg = fman->bmi_regs;
1204         irqreturn_t ret = IRQ_NONE;
1205
1206         event = ioread32be(&bmi_rg->fmbm_ievr);
1207         mask = ioread32be(&bmi_rg->fmbm_ier);
1208         event &= mask;
1209         /* clear the forced events */
1210         force = ioread32be(&bmi_rg->fmbm_ifr);
1211         if (force & event)
1212                 iowrite32be(force & ~event, &bmi_rg->fmbm_ifr);
1213         /* clear the acknowledged events */
1214         iowrite32be(event, &bmi_rg->fmbm_ievr);
1215
1216         if (event & BMI_ERR_INTR_EN_STORAGE_PROFILE_ECC)
1217                 ret = fman->exception_cb(fman, FMAN_EX_BMI_STORAGE_PROFILE_ECC);
1218         if (event & BMI_ERR_INTR_EN_LIST_RAM_ECC)
1219                 ret = fman->exception_cb(fman, FMAN_EX_BMI_LIST_RAM_ECC);
1220         if (event & BMI_ERR_INTR_EN_STATISTICS_RAM_ECC)
1221                 ret = fman->exception_cb(fman, FMAN_EX_BMI_STATISTICS_RAM_ECC);
1222         if (event & BMI_ERR_INTR_EN_DISPATCH_RAM_ECC)
1223                 ret = fman->exception_cb(fman, FMAN_EX_BMI_DISPATCH_RAM_ECC);
1224
1225         return ret;
1226 }
1227
1228 static irqreturn_t qmi_err_event(struct fman *fman)
1229 {
1230         u32 event, mask, force;
1231         struct fman_qmi_regs __iomem *qmi_rg = fman->qmi_regs;
1232         irqreturn_t ret = IRQ_NONE;
1233
1234         event = ioread32be(&qmi_rg->fmqm_eie);
1235         mask = ioread32be(&qmi_rg->fmqm_eien);
1236         event &= mask;
1237
1238         /* clear the forced events */
1239         force = ioread32be(&qmi_rg->fmqm_eif);
1240         if (force & event)
1241                 iowrite32be(force & ~event, &qmi_rg->fmqm_eif);
1242         /* clear the acknowledged events */
1243         iowrite32be(event, &qmi_rg->fmqm_eie);
1244
1245         if (event & QMI_ERR_INTR_EN_DOUBLE_ECC)
1246                 ret = fman->exception_cb(fman, FMAN_EX_QMI_DOUBLE_ECC);
1247         if (event & QMI_ERR_INTR_EN_DEQ_FROM_DEF)
1248                 ret = fman->exception_cb(fman,
1249                                          FMAN_EX_QMI_DEQ_FROM_UNKNOWN_PORTID);
1250
1251         return ret;
1252 }
1253
1254 static irqreturn_t dma_err_event(struct fman *fman)
1255 {
1256         u32 status, mask, com_id;
1257         u8 tnum, port_id, relative_port_id;
1258         u16 liodn;
1259         struct fman_dma_regs __iomem *dma_rg = fman->dma_regs;
1260         irqreturn_t ret = IRQ_NONE;
1261
1262         status = ioread32be(&dma_rg->fmdmsr);
1263         mask = ioread32be(&dma_rg->fmdmmr);
1264
1265         /* clear DMA_STATUS_BUS_ERR if mask has no DMA_MODE_BER */
1266         if ((mask & DMA_MODE_BER) != DMA_MODE_BER)
1267                 status &= ~DMA_STATUS_BUS_ERR;
1268
1269         /* clear relevant bits if mask has no DMA_MODE_ECC */
1270         if ((mask & DMA_MODE_ECC) != DMA_MODE_ECC)
1271                 status &= ~(DMA_STATUS_FM_SPDAT_ECC |
1272                             DMA_STATUS_READ_ECC |
1273                             DMA_STATUS_SYSTEM_WRITE_ECC |
1274                             DMA_STATUS_FM_WRITE_ECC);
1275
1276         /* clear set events */
1277         iowrite32be(status, &dma_rg->fmdmsr);
1278
1279         if (status & DMA_STATUS_BUS_ERR) {
1280                 u64 addr;
1281
1282                 addr = (u64)ioread32be(&dma_rg->fmdmtal);
1283                 addr |= ((u64)(ioread32be(&dma_rg->fmdmtah)) << 32);
1284
1285                 com_id = ioread32be(&dma_rg->fmdmtcid);
1286                 port_id = (u8)(((com_id & DMA_TRANSFER_PORTID_MASK) >>
1287                                DMA_TRANSFER_PORTID_SHIFT));
1288                 relative_port_id =
1289                 hw_port_id_to_sw_port_id(fman->state->rev_info.major, port_id);
1290                 tnum = (u8)((com_id & DMA_TRANSFER_TNUM_MASK) >>
1291                             DMA_TRANSFER_TNUM_SHIFT);
1292                 liodn = (u16)(com_id & DMA_TRANSFER_LIODN_MASK);
1293                 ret = fman->bus_error_cb(fman, relative_port_id, addr, tnum,
1294                                          liodn);
1295         }
1296         if (status & DMA_STATUS_FM_SPDAT_ECC)
1297                 ret = fman->exception_cb(fman, FMAN_EX_DMA_SINGLE_PORT_ECC);
1298         if (status & DMA_STATUS_READ_ECC)
1299                 ret = fman->exception_cb(fman, FMAN_EX_DMA_READ_ECC);
1300         if (status & DMA_STATUS_SYSTEM_WRITE_ECC)
1301                 ret = fman->exception_cb(fman, FMAN_EX_DMA_SYSTEM_WRITE_ECC);
1302         if (status & DMA_STATUS_FM_WRITE_ECC)
1303                 ret = fman->exception_cb(fman, FMAN_EX_DMA_FM_WRITE_ECC);
1304
1305         return ret;
1306 }
1307
1308 static irqreturn_t fpm_err_event(struct fman *fman)
1309 {
1310         u32 event;
1311         struct fman_fpm_regs __iomem *fpm_rg = fman->fpm_regs;
1312         irqreturn_t ret = IRQ_NONE;
1313
1314         event = ioread32be(&fpm_rg->fmfp_ee);
1315         /* clear the all occurred events */
1316         iowrite32be(event, &fpm_rg->fmfp_ee);
1317
1318         if ((event & FPM_EV_MASK_DOUBLE_ECC) &&
1319             (event & FPM_EV_MASK_DOUBLE_ECC_EN))
1320                 ret = fman->exception_cb(fman, FMAN_EX_FPM_DOUBLE_ECC);
1321         if ((event & FPM_EV_MASK_STALL) && (event & FPM_EV_MASK_STALL_EN))
1322                 ret = fman->exception_cb(fman, FMAN_EX_FPM_STALL_ON_TASKS);
1323         if ((event & FPM_EV_MASK_SINGLE_ECC) &&
1324             (event & FPM_EV_MASK_SINGLE_ECC_EN))
1325                 ret = fman->exception_cb(fman, FMAN_EX_FPM_SINGLE_ECC);
1326
1327         return ret;
1328 }
1329
1330 static irqreturn_t muram_err_intr(struct fman *fman)
1331 {
1332         u32 event, mask;
1333         struct fman_fpm_regs __iomem *fpm_rg = fman->fpm_regs;
1334         irqreturn_t ret = IRQ_NONE;
1335
1336         event = ioread32be(&fpm_rg->fm_rcr);
1337         mask = ioread32be(&fpm_rg->fm_rie);
1338
1339         /* clear MURAM event bit (do not clear IRAM event) */
1340         iowrite32be(event & ~FPM_RAM_IRAM_ECC, &fpm_rg->fm_rcr);
1341
1342         if ((mask & FPM_MURAM_ECC_ERR_EX_EN) && (event & FPM_RAM_MURAM_ECC))
1343                 ret = fman->exception_cb(fman, FMAN_EX_MURAM_ECC);
1344
1345         return ret;
1346 }
1347
1348 static irqreturn_t qmi_event(struct fman *fman)
1349 {
1350         u32 event, mask, force;
1351         struct fman_qmi_regs __iomem *qmi_rg = fman->qmi_regs;
1352         irqreturn_t ret = IRQ_NONE;
1353
1354         event = ioread32be(&qmi_rg->fmqm_ie);
1355         mask = ioread32be(&qmi_rg->fmqm_ien);
1356         event &= mask;
1357         /* clear the forced events */
1358         force = ioread32be(&qmi_rg->fmqm_if);
1359         if (force & event)
1360                 iowrite32be(force & ~event, &qmi_rg->fmqm_if);
1361         /* clear the acknowledged events */
1362         iowrite32be(event, &qmi_rg->fmqm_ie);
1363
1364         if (event & QMI_INTR_EN_SINGLE_ECC)
1365                 ret = fman->exception_cb(fman, FMAN_EX_QMI_SINGLE_ECC);
1366
1367         return ret;
1368 }
1369
1370 static void enable_time_stamp(struct fman *fman)
1371 {
1372         struct fman_fpm_regs __iomem *fpm_rg = fman->fpm_regs;
1373         u16 fm_clk_freq = fman->state->fm_clk_freq;
1374         u32 tmp, intgr, ts_freq, frac;
1375
1376         ts_freq = (u32)(1 << fman->state->count1_micro_bit);
1377         /* configure timestamp so that bit 8 will count 1 microsecond
1378          * Find effective count rate at TIMESTAMP least significant bits:
1379          * Effective_Count_Rate = 1MHz x 2^8 = 256MHz
1380          * Find frequency ratio between effective count rate and the clock:
1381          * Effective_Count_Rate / CLK e.g. for 600 MHz clock:
1382          * 256/600 = 0.4266666...
1383          */
1384
1385         intgr = ts_freq / fm_clk_freq;
1386         /* we multiply by 2^16 to keep the fraction of the division
1387          * we do not div back, since we write this value as a fraction
1388          * see spec
1389          */
1390
1391         frac = ((ts_freq << 16) - (intgr << 16) * fm_clk_freq) / fm_clk_freq;
1392         /* we check remainder of the division in order to round up if not int */
1393         if (((ts_freq << 16) - (intgr << 16) * fm_clk_freq) % fm_clk_freq)
1394                 frac++;
1395
1396         tmp = (intgr << FPM_TS_INT_SHIFT) | (u16)frac;
1397         iowrite32be(tmp, &fpm_rg->fmfp_tsc2);
1398
1399         /* enable timestamp with original clock */
1400         iowrite32be(FPM_TS_CTL_EN, &fpm_rg->fmfp_tsc1);
1401         fman->state->enabled_time_stamp = true;
1402 }
1403
1404 static int clear_iram(struct fman *fman)
1405 {
1406         struct fman_iram_regs __iomem *iram;
1407         int i, count;
1408
1409         iram = fman->base_addr + IMEM_OFFSET;
1410
1411         /* Enable the auto-increment */
1412         iowrite32be(IRAM_IADD_AIE, &iram->iadd);
1413         count = 100;
1414         do {
1415                 udelay(1);
1416         } while ((ioread32be(&iram->iadd) != IRAM_IADD_AIE) && --count);
1417         if (count == 0)
1418                 return -EBUSY;
1419
1420         for (i = 0; i < (fman->state->fm_iram_size / 4); i++)
1421                 iowrite32be(0xffffffff, &iram->idata);
1422
1423         iowrite32be(fman->state->fm_iram_size - 4, &iram->iadd);
1424         count = 100;
1425         do {
1426                 udelay(1);
1427         } while ((ioread32be(&iram->idata) != 0xffffffff) && --count);
1428         if (count == 0)
1429                 return -EBUSY;
1430
1431         return 0;
1432 }
1433
1434 static u32 get_exception_flag(enum fman_exceptions exception)
1435 {
1436         u32 bit_mask;
1437
1438         switch (exception) {
1439         case FMAN_EX_DMA_BUS_ERROR:
1440                 bit_mask = EX_DMA_BUS_ERROR;
1441                 break;
1442         case FMAN_EX_DMA_SINGLE_PORT_ECC:
1443                 bit_mask = EX_DMA_SINGLE_PORT_ECC;
1444                 break;
1445         case FMAN_EX_DMA_READ_ECC:
1446                 bit_mask = EX_DMA_READ_ECC;
1447                 break;
1448         case FMAN_EX_DMA_SYSTEM_WRITE_ECC:
1449                 bit_mask = EX_DMA_SYSTEM_WRITE_ECC;
1450                 break;
1451         case FMAN_EX_DMA_FM_WRITE_ECC:
1452                 bit_mask = EX_DMA_FM_WRITE_ECC;
1453                 break;
1454         case FMAN_EX_FPM_STALL_ON_TASKS:
1455                 bit_mask = EX_FPM_STALL_ON_TASKS;
1456                 break;
1457         case FMAN_EX_FPM_SINGLE_ECC:
1458                 bit_mask = EX_FPM_SINGLE_ECC;
1459                 break;
1460         case FMAN_EX_FPM_DOUBLE_ECC:
1461                 bit_mask = EX_FPM_DOUBLE_ECC;
1462                 break;
1463         case FMAN_EX_QMI_SINGLE_ECC:
1464                 bit_mask = EX_QMI_SINGLE_ECC;
1465                 break;
1466         case FMAN_EX_QMI_DOUBLE_ECC:
1467                 bit_mask = EX_QMI_DOUBLE_ECC;
1468                 break;
1469         case FMAN_EX_QMI_DEQ_FROM_UNKNOWN_PORTID:
1470                 bit_mask = EX_QMI_DEQ_FROM_UNKNOWN_PORTID;
1471                 break;
1472         case FMAN_EX_BMI_LIST_RAM_ECC:
1473                 bit_mask = EX_BMI_LIST_RAM_ECC;
1474                 break;
1475         case FMAN_EX_BMI_STORAGE_PROFILE_ECC:
1476                 bit_mask = EX_BMI_STORAGE_PROFILE_ECC;
1477                 break;
1478         case FMAN_EX_BMI_STATISTICS_RAM_ECC:
1479                 bit_mask = EX_BMI_STATISTICS_RAM_ECC;
1480                 break;
1481         case FMAN_EX_BMI_DISPATCH_RAM_ECC:
1482                 bit_mask = EX_BMI_DISPATCH_RAM_ECC;
1483                 break;
1484         case FMAN_EX_MURAM_ECC:
1485                 bit_mask = EX_MURAM_ECC;
1486                 break;
1487         default:
1488                 bit_mask = 0;
1489                 break;
1490         }
1491
1492         return bit_mask;
1493 }
1494
1495 static int get_module_event(enum fman_event_modules module, u8 mod_id,
1496                             enum fman_intr_type intr_type)
1497 {
1498         int event;
1499
1500         switch (module) {
1501         case FMAN_MOD_MAC:
1502                 if (intr_type == FMAN_INTR_TYPE_ERR)
1503                         event = FMAN_EV_ERR_MAC0 + mod_id;
1504                 else
1505                         event = FMAN_EV_MAC0 + mod_id;
1506                 break;
1507         case FMAN_MOD_FMAN_CTRL:
1508                 if (intr_type == FMAN_INTR_TYPE_ERR)
1509                         event = FMAN_EV_CNT;
1510                 else
1511                         event = (FMAN_EV_FMAN_CTRL_0 + mod_id);
1512                 break;
1513         case FMAN_MOD_DUMMY_LAST:
1514                 event = FMAN_EV_CNT;
1515                 break;
1516         default:
1517                 event = FMAN_EV_CNT;
1518                 break;
1519         }
1520
1521         return event;
1522 }
1523
1524 static int set_size_of_fifo(struct fman *fman, u8 port_id, u32 *size_of_fifo,
1525                             u32 *extra_size_of_fifo)
1526 {
1527         struct fman_bmi_regs __iomem *bmi_rg = fman->bmi_regs;
1528         u32 fifo = *size_of_fifo;
1529         u32 extra_fifo = *extra_size_of_fifo;
1530         u32 tmp;
1531
1532         /* if this is the first time a port requires extra_fifo_pool_size,
1533          * the total extra_fifo_pool_size must be initialized to 1 buffer per
1534          * port
1535          */
1536         if (extra_fifo && !fman->state->extra_fifo_pool_size)
1537                 fman->state->extra_fifo_pool_size =
1538                         fman->state->num_of_rx_ports * FMAN_BMI_FIFO_UNITS;
1539
1540         fman->state->extra_fifo_pool_size =
1541                 max(fman->state->extra_fifo_pool_size, extra_fifo);
1542
1543         /* check that there are enough uncommitted fifo size */
1544         if ((fman->state->accumulated_fifo_size + fifo) >
1545             (fman->state->total_fifo_size -
1546             fman->state->extra_fifo_pool_size)) {
1547                 dev_err(fman->dev, "%s: Requested fifo size and extra size exceed total FIFO size.\n",
1548                         __func__);
1549                 return -EAGAIN;
1550         }
1551
1552         /* Read, modify and write to HW */
1553         tmp = (fifo / FMAN_BMI_FIFO_UNITS - 1) |
1554                ((extra_fifo / FMAN_BMI_FIFO_UNITS) <<
1555                BMI_EXTRA_FIFO_SIZE_SHIFT);
1556         iowrite32be(tmp, &bmi_rg->fmbm_pfs[port_id - 1]);
1557
1558         /* update accumulated */
1559         fman->state->accumulated_fifo_size += fifo;
1560
1561         return 0;
1562 }
1563
1564 static int set_num_of_tasks(struct fman *fman, u8 port_id, u8 *num_of_tasks,
1565                             u8 *num_of_extra_tasks)
1566 {
1567         struct fman_bmi_regs __iomem *bmi_rg = fman->bmi_regs;
1568         u8 tasks = *num_of_tasks;
1569         u8 extra_tasks = *num_of_extra_tasks;
1570         u32 tmp;
1571
1572         if (extra_tasks)
1573                 fman->state->extra_tasks_pool_size =
1574                 max(fman->state->extra_tasks_pool_size, extra_tasks);
1575
1576         /* check that there are enough uncommitted tasks */
1577         if ((fman->state->accumulated_num_of_tasks + tasks) >
1578             (fman->state->total_num_of_tasks -
1579              fman->state->extra_tasks_pool_size)) {
1580                 dev_err(fman->dev, "%s: Requested num_of_tasks and extra tasks pool for fm%d exceed total num_of_tasks.\n",
1581                         __func__, fman->state->fm_id);
1582                 return -EAGAIN;
1583         }
1584         /* update accumulated */
1585         fman->state->accumulated_num_of_tasks += tasks;
1586
1587         /* Write to HW */
1588         tmp = ioread32be(&bmi_rg->fmbm_pp[port_id - 1]) &
1589             ~(BMI_NUM_OF_TASKS_MASK | BMI_NUM_OF_EXTRA_TASKS_MASK);
1590         tmp |= ((u32)((tasks - 1) << BMI_NUM_OF_TASKS_SHIFT) |
1591                 (u32)(extra_tasks << BMI_EXTRA_NUM_OF_TASKS_SHIFT));
1592         iowrite32be(tmp, &bmi_rg->fmbm_pp[port_id - 1]);
1593
1594         return 0;
1595 }
1596
1597 static int set_num_of_open_dmas(struct fman *fman, u8 port_id,
1598                                 u8 *num_of_open_dmas,
1599                                 u8 *num_of_extra_open_dmas)
1600 {
1601         struct fman_bmi_regs __iomem *bmi_rg = fman->bmi_regs;
1602         u8 open_dmas = *num_of_open_dmas;
1603         u8 extra_open_dmas = *num_of_extra_open_dmas;
1604         u8 total_num_dmas = 0, current_val = 0, current_extra_val = 0;
1605         u32 tmp;
1606
1607         if (!open_dmas) {
1608                 /* Configuration according to values in the HW.
1609                  * read the current number of open Dma's
1610                  */
1611                 tmp = ioread32be(&bmi_rg->fmbm_pp[port_id - 1]);
1612                 current_extra_val = (u8)((tmp & BMI_NUM_OF_EXTRA_DMAS_MASK) >>
1613                                          BMI_EXTRA_NUM_OF_DMAS_SHIFT);
1614
1615                 tmp = ioread32be(&bmi_rg->fmbm_pp[port_id - 1]);
1616                 current_val = (u8)(((tmp & BMI_NUM_OF_DMAS_MASK) >>
1617                                    BMI_NUM_OF_DMAS_SHIFT) + 1);
1618
1619                 /* This is the first configuration and user did not
1620                  * specify value (!open_dmas), reset values will be used
1621                  * and we just save these values for resource management
1622                  */
1623                 fman->state->extra_open_dmas_pool_size =
1624                         (u8)max(fman->state->extra_open_dmas_pool_size,
1625                                 current_extra_val);
1626                 fman->state->accumulated_num_of_open_dmas += current_val;
1627                 *num_of_open_dmas = current_val;
1628                 *num_of_extra_open_dmas = current_extra_val;
1629                 return 0;
1630         }
1631
1632         if (extra_open_dmas > current_extra_val)
1633                 fman->state->extra_open_dmas_pool_size =
1634                     (u8)max(fman->state->extra_open_dmas_pool_size,
1635                             extra_open_dmas);
1636
1637         if ((fman->state->rev_info.major < 6) &&
1638             (fman->state->accumulated_num_of_open_dmas - current_val +
1639              open_dmas > fman->state->max_num_of_open_dmas)) {
1640                 dev_err(fman->dev, "%s: Requested num_of_open_dmas for fm%d exceeds total num_of_open_dmas.\n",
1641                         __func__, fman->state->fm_id);
1642                 return -EAGAIN;
1643         } else if ((fman->state->rev_info.major >= 6) &&
1644                    !((fman->state->rev_info.major == 6) &&
1645                    (fman->state->rev_info.minor == 0)) &&
1646                    (fman->state->accumulated_num_of_open_dmas -
1647                    current_val + open_dmas >
1648                    fman->state->dma_thresh_max_commq + 1)) {
1649                 dev_err(fman->dev, "%s: Requested num_of_open_dmas for fm%d exceeds DMA Command queue (%d)\n",
1650                         __func__, fman->state->fm_id,
1651                        fman->state->dma_thresh_max_commq + 1);
1652                 return -EAGAIN;
1653         }
1654
1655         WARN_ON(fman->state->accumulated_num_of_open_dmas < current_val);
1656         /* update acummulated */
1657         fman->state->accumulated_num_of_open_dmas -= current_val;
1658         fman->state->accumulated_num_of_open_dmas += open_dmas;
1659
1660         if (fman->state->rev_info.major < 6)
1661                 total_num_dmas =
1662                     (u8)(fman->state->accumulated_num_of_open_dmas +
1663                     fman->state->extra_open_dmas_pool_size);
1664
1665         /* calculate reg */
1666         tmp = ioread32be(&bmi_rg->fmbm_pp[port_id - 1]) &
1667             ~(BMI_NUM_OF_DMAS_MASK | BMI_NUM_OF_EXTRA_DMAS_MASK);
1668         tmp |= (u32)(((open_dmas - 1) << BMI_NUM_OF_DMAS_SHIFT) |
1669                            (extra_open_dmas << BMI_EXTRA_NUM_OF_DMAS_SHIFT));
1670         iowrite32be(tmp, &bmi_rg->fmbm_pp[port_id - 1]);
1671
1672         /* update total num of DMA's with committed number of open DMAS,
1673          * and max uncommitted pool.
1674          */
1675         if (total_num_dmas) {
1676                 tmp = ioread32be(&bmi_rg->fmbm_cfg2) & ~BMI_CFG2_DMAS_MASK;
1677                 tmp |= (u32)(total_num_dmas - 1) << BMI_CFG2_DMAS_SHIFT;
1678                 iowrite32be(tmp, &bmi_rg->fmbm_cfg2);
1679         }
1680
1681         return 0;
1682 }
1683
1684 static int fman_config(struct fman *fman)
1685 {
1686         void __iomem *base_addr;
1687         int err;
1688
1689         base_addr = fman->dts_params.base_addr;
1690
1691         fman->state = kzalloc(sizeof(*fman->state), GFP_KERNEL);
1692         if (!fman->state)
1693                 goto err_fm_state;
1694
1695         /* Allocate the FM driver's parameters structure */
1696         fman->cfg = kzalloc(sizeof(*fman->cfg), GFP_KERNEL);
1697         if (!fman->cfg)
1698                 goto err_fm_drv;
1699
1700         /* Initialize MURAM block */
1701         fman->muram =
1702                 fman_muram_init(fman->dts_params.muram_res.start,
1703                                 resource_size(&fman->dts_params.muram_res));
1704         if (!fman->muram)
1705                 goto err_fm_soc_specific;
1706
1707         /* Initialize FM parameters which will be kept by the driver */
1708         fman->state->fm_id = fman->dts_params.id;
1709         fman->state->fm_clk_freq = fman->dts_params.clk_freq;
1710         fman->state->qman_channel_base = fman->dts_params.qman_channel_base;
1711         fman->state->num_of_qman_channels =
1712                 fman->dts_params.num_of_qman_channels;
1713         fman->state->res = fman->dts_params.res;
1714         fman->exception_cb = fman_exceptions;
1715         fman->bus_error_cb = fman_bus_error;
1716         fman->fpm_regs = base_addr + FPM_OFFSET;
1717         fman->bmi_regs = base_addr + BMI_OFFSET;
1718         fman->qmi_regs = base_addr + QMI_OFFSET;
1719         fman->dma_regs = base_addr + DMA_OFFSET;
1720         fman->hwp_regs = base_addr + HWP_OFFSET;
1721         fman->kg_regs = base_addr + KG_OFFSET;
1722         fman->base_addr = base_addr;
1723
1724         spin_lock_init(&fman->spinlock);
1725         fman_defconfig(fman->cfg);
1726
1727         fman->state->extra_fifo_pool_size = 0;
1728         fman->state->exceptions = (EX_DMA_BUS_ERROR                 |
1729                                         EX_DMA_READ_ECC              |
1730                                         EX_DMA_SYSTEM_WRITE_ECC      |
1731                                         EX_DMA_FM_WRITE_ECC          |
1732                                         EX_FPM_STALL_ON_TASKS        |
1733                                         EX_FPM_SINGLE_ECC            |
1734                                         EX_FPM_DOUBLE_ECC            |
1735                                         EX_QMI_DEQ_FROM_UNKNOWN_PORTID |
1736                                         EX_BMI_LIST_RAM_ECC          |
1737                                         EX_BMI_STORAGE_PROFILE_ECC   |
1738                                         EX_BMI_STATISTICS_RAM_ECC    |
1739                                         EX_MURAM_ECC                 |
1740                                         EX_BMI_DISPATCH_RAM_ECC      |
1741                                         EX_QMI_DOUBLE_ECC            |
1742                                         EX_QMI_SINGLE_ECC);
1743
1744         /* Read FMan revision for future use*/
1745         fman_get_revision(fman, &fman->state->rev_info);
1746
1747         err = fill_soc_specific_params(fman->state);
1748         if (err)
1749                 goto err_fm_soc_specific;
1750
1751         /* FM_AID_MODE_NO_TNUM_SW005 Errata workaround */
1752         if (fman->state->rev_info.major >= 6)
1753                 fman->cfg->dma_aid_mode = FMAN_DMA_AID_OUT_PORT_ID;
1754
1755         fman->cfg->qmi_def_tnums_thresh = fman->state->qmi_def_tnums_thresh;
1756
1757         fman->state->total_num_of_tasks =
1758         (u8)DFLT_TOTAL_NUM_OF_TASKS(fman->state->rev_info.major,
1759                                     fman->state->rev_info.minor,
1760                                     fman->state->bmi_max_num_of_tasks);
1761
1762         if (fman->state->rev_info.major < 6) {
1763                 fman->cfg->dma_comm_qtsh_clr_emer =
1764                 (u8)DFLT_DMA_COMM_Q_LOW(fman->state->rev_info.major,
1765                                         fman->state->dma_thresh_max_commq);
1766
1767                 fman->cfg->dma_comm_qtsh_asrt_emer =
1768                 (u8)DFLT_DMA_COMM_Q_HIGH(fman->state->rev_info.major,
1769                                          fman->state->dma_thresh_max_commq);
1770
1771                 fman->cfg->dma_cam_num_of_entries =
1772                 DFLT_DMA_CAM_NUM_OF_ENTRIES(fman->state->rev_info.major);
1773
1774                 fman->cfg->dma_read_buf_tsh_clr_emer =
1775                 DFLT_DMA_READ_INT_BUF_LOW(fman->state->dma_thresh_max_buf);
1776
1777                 fman->cfg->dma_read_buf_tsh_asrt_emer =
1778                 DFLT_DMA_READ_INT_BUF_HIGH(fman->state->dma_thresh_max_buf);
1779
1780                 fman->cfg->dma_write_buf_tsh_clr_emer =
1781                 DFLT_DMA_WRITE_INT_BUF_LOW(fman->state->dma_thresh_max_buf);
1782
1783                 fman->cfg->dma_write_buf_tsh_asrt_emer =
1784                 DFLT_DMA_WRITE_INT_BUF_HIGH(fman->state->dma_thresh_max_buf);
1785
1786                 fman->cfg->dma_axi_dbg_num_of_beats =
1787                 DFLT_AXI_DBG_NUM_OF_BEATS;
1788         }
1789
1790         return 0;
1791
1792 err_fm_soc_specific:
1793         kfree(fman->cfg);
1794 err_fm_drv:
1795         kfree(fman->state);
1796 err_fm_state:
1797         kfree(fman);
1798         return -EINVAL;
1799 }
1800
1801 static int fman_reset(struct fman *fman)
1802 {
1803         u32 count;
1804         int err = 0;
1805
1806         if (fman->state->rev_info.major < 6) {
1807                 iowrite32be(FPM_RSTC_FM_RESET, &fman->fpm_regs->fm_rstc);
1808                 /* Wait for reset completion */
1809                 count = 100;
1810                 do {
1811                         udelay(1);
1812                 } while (((ioread32be(&fman->fpm_regs->fm_rstc)) &
1813                          FPM_RSTC_FM_RESET) && --count);
1814                 if (count == 0)
1815                         err = -EBUSY;
1816
1817                 goto _return;
1818         } else {
1819 #ifdef CONFIG_PPC
1820                 struct device_node *guts_node;
1821                 struct ccsr_guts __iomem *guts_regs;
1822                 u32 devdisr2, reg;
1823
1824                 /* Errata A007273 */
1825                 guts_node =
1826                         of_find_compatible_node(NULL, NULL,
1827                                                 "fsl,qoriq-device-config-2.0");
1828                 if (!guts_node) {
1829                         dev_err(fman->dev, "%s: Couldn't find guts node\n",
1830                                 __func__);
1831                         goto guts_node;
1832                 }
1833
1834                 guts_regs = of_iomap(guts_node, 0);
1835                 if (!guts_regs) {
1836                         dev_err(fman->dev, "%s: Couldn't map %pOF regs\n",
1837                                 __func__, guts_node);
1838                         goto guts_regs;
1839                 }
1840 #define FMAN1_ALL_MACS_MASK     0xFCC00000
1841 #define FMAN2_ALL_MACS_MASK     0x000FCC00
1842                 /* Read current state */
1843                 devdisr2 = ioread32be(&guts_regs->devdisr2);
1844                 if (fman->dts_params.id == 0)
1845                         reg = devdisr2 & ~FMAN1_ALL_MACS_MASK;
1846                 else
1847                         reg = devdisr2 & ~FMAN2_ALL_MACS_MASK;
1848
1849                 /* Enable all MACs */
1850                 iowrite32be(reg, &guts_regs->devdisr2);
1851 #endif
1852
1853                 /* Perform FMan reset */
1854                 iowrite32be(FPM_RSTC_FM_RESET, &fman->fpm_regs->fm_rstc);
1855
1856                 /* Wait for reset completion */
1857                 count = 100;
1858                 do {
1859                         udelay(1);
1860                 } while (((ioread32be(&fman->fpm_regs->fm_rstc)) &
1861                          FPM_RSTC_FM_RESET) && --count);
1862                 if (count == 0) {
1863 #ifdef CONFIG_PPC
1864                         iounmap(guts_regs);
1865                         of_node_put(guts_node);
1866 #endif
1867                         err = -EBUSY;
1868                         goto _return;
1869                 }
1870 #ifdef CONFIG_PPC
1871
1872                 /* Restore devdisr2 value */
1873                 iowrite32be(devdisr2, &guts_regs->devdisr2);
1874
1875                 iounmap(guts_regs);
1876                 of_node_put(guts_node);
1877 #endif
1878
1879                 goto _return;
1880
1881 #ifdef CONFIG_PPC
1882 guts_regs:
1883                 of_node_put(guts_node);
1884 guts_node:
1885                 dev_dbg(fman->dev, "%s: Didn't perform FManV3 reset due to Errata A007273!\n",
1886                         __func__);
1887 #endif
1888         }
1889 _return:
1890         return err;
1891 }
1892
1893 static int fman_init(struct fman *fman)
1894 {
1895         struct fman_cfg *cfg = NULL;
1896         int err = 0, i, count;
1897
1898         if (is_init_done(fman->cfg))
1899                 return -EINVAL;
1900
1901         fman->state->count1_micro_bit = FM_TIMESTAMP_1_USEC_BIT;
1902
1903         cfg = fman->cfg;
1904
1905         /* clear revision-dependent non existing exception */
1906         if (fman->state->rev_info.major < 6)
1907                 fman->state->exceptions &= ~FMAN_EX_BMI_DISPATCH_RAM_ECC;
1908
1909         if (fman->state->rev_info.major >= 6)
1910                 fman->state->exceptions &= ~FMAN_EX_QMI_SINGLE_ECC;
1911
1912         /* clear CPG */
1913         memset_io((void __iomem *)(fman->base_addr + CGP_OFFSET), 0,
1914                   fman->state->fm_port_num_of_cg);
1915
1916         /* Save LIODN info before FMan reset
1917          * Skipping non-existent port 0 (i = 1)
1918          */
1919         for (i = 1; i < FMAN_LIODN_TBL; i++) {
1920                 u32 liodn_base;
1921
1922                 fman->liodn_offset[i] =
1923                         ioread32be(&fman->bmi_regs->fmbm_spliodn[i - 1]);
1924                 if (!IS_ENABLED(CONFIG_FSL_PAMU))
1925                         continue;
1926                 liodn_base = ioread32be(&fman->dma_regs->fmdmplr[i / 2]);
1927                 if (i % 2) {
1928                         /* FMDM_PLR LSB holds LIODN base for odd ports */
1929                         liodn_base &= DMA_LIODN_BASE_MASK;
1930                 } else {
1931                         /* FMDM_PLR MSB holds LIODN base for even ports */
1932                         liodn_base >>= DMA_LIODN_SHIFT;
1933                         liodn_base &= DMA_LIODN_BASE_MASK;
1934                 }
1935                 fman->liodn_base[i] = liodn_base;
1936         }
1937
1938         err = fman_reset(fman);
1939         if (err)
1940                 return err;
1941
1942         if (ioread32be(&fman->qmi_regs->fmqm_gs) & QMI_GS_HALT_NOT_BUSY) {
1943                 resume(fman->fpm_regs);
1944                 /* Wait until QMI is not in halt not busy state */
1945                 count = 100;
1946                 do {
1947                         udelay(1);
1948                 } while (((ioread32be(&fman->qmi_regs->fmqm_gs)) &
1949                          QMI_GS_HALT_NOT_BUSY) && --count);
1950                 if (count == 0)
1951                         dev_warn(fman->dev, "%s: QMI is in halt not busy state\n",
1952                                  __func__);
1953         }
1954
1955         if (clear_iram(fman) != 0)
1956                 return -EINVAL;
1957
1958         cfg->exceptions = fman->state->exceptions;
1959
1960         /* Init DMA Registers */
1961
1962         err = dma_init(fman);
1963         if (err != 0) {
1964                 free_init_resources(fman);
1965                 return err;
1966         }
1967
1968         /* Init FPM Registers */
1969         fpm_init(fman->fpm_regs, fman->cfg);
1970
1971         /* define common resources */
1972         /* allocate MURAM for FIFO according to total size */
1973         fman->fifo_offset = fman_muram_alloc(fman->muram,
1974                                              fman->state->total_fifo_size);
1975         if (IS_ERR_VALUE(fman->fifo_offset)) {
1976                 free_init_resources(fman);
1977                 dev_err(fman->dev, "%s: MURAM alloc for BMI FIFO failed\n",
1978                         __func__);
1979                 return -ENOMEM;
1980         }
1981
1982         cfg->fifo_base_addr = fman->fifo_offset;
1983         cfg->total_fifo_size = fman->state->total_fifo_size;
1984         cfg->total_num_of_tasks = fman->state->total_num_of_tasks;
1985         cfg->clk_freq = fman->state->fm_clk_freq;
1986
1987         /* Init BMI Registers */
1988         bmi_init(fman->bmi_regs, fman->cfg);
1989
1990         /* Init QMI Registers */
1991         qmi_init(fman->qmi_regs, fman->cfg);
1992
1993         /* Init HW Parser */
1994         hwp_init(fman->hwp_regs);
1995
1996         /* Init KeyGen */
1997         fman->keygen = keygen_init(fman->kg_regs);
1998         if (!fman->keygen)
1999                 return -EINVAL;
2000
2001         err = enable(fman, cfg);
2002         if (err != 0)
2003                 return err;
2004
2005         enable_time_stamp(fman);
2006
2007         kfree(fman->cfg);
2008         fman->cfg = NULL;
2009
2010         return 0;
2011 }
2012
2013 static int fman_set_exception(struct fman *fman,
2014                               enum fman_exceptions exception, bool enable)
2015 {
2016         u32 bit_mask = 0;
2017
2018         if (!is_init_done(fman->cfg))
2019                 return -EINVAL;
2020
2021         bit_mask = get_exception_flag(exception);
2022         if (bit_mask) {
2023                 if (enable)
2024                         fman->state->exceptions |= bit_mask;
2025                 else
2026                         fman->state->exceptions &= ~bit_mask;
2027         } else {
2028                 dev_err(fman->dev, "%s: Undefined exception (%d)\n",
2029                         __func__, exception);
2030                 return -EINVAL;
2031         }
2032
2033         return set_exception(fman, exception, enable);
2034 }
2035
2036 /**
2037  * fman_register_intr
2038  * @fman:       A Pointer to FMan device
2039  * @module:     Calling module
2040  * @mod_id:     Module id (if more than 1 exists, '0' if not)
2041  * @intr_type:  Interrupt type (error/normal) selection.
2042  * @isr_cb:     The interrupt service routine.
2043  * @src_arg:    Argument to be passed to isr_cb.
2044  *
2045  * Used to register an event handler to be processed by FMan
2046  *
2047  * Return: 0 on success; Error code otherwise.
2048  */
2049 void fman_register_intr(struct fman *fman, enum fman_event_modules module,
2050                         u8 mod_id, enum fman_intr_type intr_type,
2051                         void (*isr_cb)(void *src_arg), void *src_arg)
2052 {
2053         int event = 0;
2054
2055         event = get_module_event(module, mod_id, intr_type);
2056         WARN_ON(event >= FMAN_EV_CNT);
2057
2058         /* register in local FM structure */
2059         fman->intr_mng[event].isr_cb = isr_cb;
2060         fman->intr_mng[event].src_handle = src_arg;
2061 }
2062 EXPORT_SYMBOL(fman_register_intr);
2063
2064 /**
2065  * fman_unregister_intr
2066  * @fman:       A Pointer to FMan device
2067  * @module:     Calling module
2068  * @mod_id:     Module id (if more than 1 exists, '0' if not)
2069  * @intr_type:  Interrupt type (error/normal) selection.
2070  *
2071  * Used to unregister an event handler to be processed by FMan
2072  *
2073  * Return: 0 on success; Error code otherwise.
2074  */
2075 void fman_unregister_intr(struct fman *fman, enum fman_event_modules module,
2076                           u8 mod_id, enum fman_intr_type intr_type)
2077 {
2078         int event = 0;
2079
2080         event = get_module_event(module, mod_id, intr_type);
2081         WARN_ON(event >= FMAN_EV_CNT);
2082
2083         fman->intr_mng[event].isr_cb = NULL;
2084         fman->intr_mng[event].src_handle = NULL;
2085 }
2086 EXPORT_SYMBOL(fman_unregister_intr);
2087
2088 /**
2089  * fman_set_port_params
2090  * @fman:               A Pointer to FMan device
2091  * @port_params:        Port parameters
2092  *
2093  * Used by FMan Port to pass parameters to the FMan
2094  *
2095  * Return: 0 on success; Error code otherwise.
2096  */
2097 int fman_set_port_params(struct fman *fman,
2098                          struct fman_port_init_params *port_params)
2099 {
2100         int err;
2101         unsigned long flags;
2102         u8 port_id = port_params->port_id, mac_id;
2103
2104         spin_lock_irqsave(&fman->spinlock, flags);
2105
2106         err = set_num_of_tasks(fman, port_params->port_id,
2107                                &port_params->num_of_tasks,
2108                                &port_params->num_of_extra_tasks);
2109         if (err)
2110                 goto return_err;
2111
2112         /* TX Ports */
2113         if (port_params->port_type != FMAN_PORT_TYPE_RX) {
2114                 u32 enq_th, deq_th, reg;
2115
2116                 /* update qmi ENQ/DEQ threshold */
2117                 fman->state->accumulated_num_of_deq_tnums +=
2118                         port_params->deq_pipeline_depth;
2119                 enq_th = (ioread32be(&fman->qmi_regs->fmqm_gc) &
2120                           QMI_CFG_ENQ_MASK) >> QMI_CFG_ENQ_SHIFT;
2121                 /* if enq_th is too big, we reduce it to the max value
2122                  * that is still 0
2123                  */
2124                 if (enq_th >= (fman->state->qmi_max_num_of_tnums -
2125                     fman->state->accumulated_num_of_deq_tnums)) {
2126                         enq_th =
2127                         fman->state->qmi_max_num_of_tnums -
2128                         fman->state->accumulated_num_of_deq_tnums - 1;
2129
2130                         reg = ioread32be(&fman->qmi_regs->fmqm_gc);
2131                         reg &= ~QMI_CFG_ENQ_MASK;
2132                         reg |= (enq_th << QMI_CFG_ENQ_SHIFT);
2133                         iowrite32be(reg, &fman->qmi_regs->fmqm_gc);
2134                 }
2135
2136                 deq_th = ioread32be(&fman->qmi_regs->fmqm_gc) &
2137                                     QMI_CFG_DEQ_MASK;
2138                 /* if deq_th is too small, we enlarge it to the min
2139                  * value that is still 0.
2140                  * depTh may not be larger than 63
2141                  * (fman->state->qmi_max_num_of_tnums-1).
2142                  */
2143                 if ((deq_th <= fman->state->accumulated_num_of_deq_tnums) &&
2144                     (deq_th < fman->state->qmi_max_num_of_tnums - 1)) {
2145                         deq_th = fman->state->accumulated_num_of_deq_tnums + 1;
2146                         reg = ioread32be(&fman->qmi_regs->fmqm_gc);
2147                         reg &= ~QMI_CFG_DEQ_MASK;
2148                         reg |= deq_th;
2149                         iowrite32be(reg, &fman->qmi_regs->fmqm_gc);
2150                 }
2151         }
2152
2153         err = set_size_of_fifo(fman, port_params->port_id,
2154                                &port_params->size_of_fifo,
2155                                &port_params->extra_size_of_fifo);
2156         if (err)
2157                 goto return_err;
2158
2159         err = set_num_of_open_dmas(fman, port_params->port_id,
2160                                    &port_params->num_of_open_dmas,
2161                                    &port_params->num_of_extra_open_dmas);
2162         if (err)
2163                 goto return_err;
2164
2165         set_port_liodn(fman, port_id, fman->liodn_base[port_id],
2166                        fman->liodn_offset[port_id]);
2167
2168         if (fman->state->rev_info.major < 6)
2169                 set_port_order_restoration(fman->fpm_regs, port_id);
2170
2171         mac_id = hw_port_id_to_sw_port_id(fman->state->rev_info.major, port_id);
2172
2173         if (port_params->max_frame_length >= fman->state->mac_mfl[mac_id]) {
2174                 fman->state->port_mfl[mac_id] = port_params->max_frame_length;
2175         } else {
2176                 dev_warn(fman->dev, "%s: Port (%d) max_frame_length is smaller than MAC (%d) current MTU\n",
2177                          __func__, port_id, mac_id);
2178                 err = -EINVAL;
2179                 goto return_err;
2180         }
2181
2182         spin_unlock_irqrestore(&fman->spinlock, flags);
2183
2184         return 0;
2185
2186 return_err:
2187         spin_unlock_irqrestore(&fman->spinlock, flags);
2188         return err;
2189 }
2190 EXPORT_SYMBOL(fman_set_port_params);
2191
2192 /**
2193  * fman_reset_mac
2194  * @fman:       A Pointer to FMan device
2195  * @mac_id:     MAC id to be reset
2196  *
2197  * Reset a specific MAC
2198  *
2199  * Return: 0 on success; Error code otherwise.
2200  */
2201 int fman_reset_mac(struct fman *fman, u8 mac_id)
2202 {
2203         struct fman_fpm_regs __iomem *fpm_rg = fman->fpm_regs;
2204         u32 msk, timeout = 100;
2205
2206         if (fman->state->rev_info.major >= 6) {
2207                 dev_err(fman->dev, "%s: FMan MAC reset no available for FMan V3!\n",
2208                         __func__);
2209                 return -EINVAL;
2210         }
2211
2212         /* Get the relevant bit mask */
2213         switch (mac_id) {
2214         case 0:
2215                 msk = FPM_RSTC_MAC0_RESET;
2216                 break;
2217         case 1:
2218                 msk = FPM_RSTC_MAC1_RESET;
2219                 break;
2220         case 2:
2221                 msk = FPM_RSTC_MAC2_RESET;
2222                 break;
2223         case 3:
2224                 msk = FPM_RSTC_MAC3_RESET;
2225                 break;
2226         case 4:
2227                 msk = FPM_RSTC_MAC4_RESET;
2228                 break;
2229         case 5:
2230                 msk = FPM_RSTC_MAC5_RESET;
2231                 break;
2232         case 6:
2233                 msk = FPM_RSTC_MAC6_RESET;
2234                 break;
2235         case 7:
2236                 msk = FPM_RSTC_MAC7_RESET;
2237                 break;
2238         case 8:
2239                 msk = FPM_RSTC_MAC8_RESET;
2240                 break;
2241         case 9:
2242                 msk = FPM_RSTC_MAC9_RESET;
2243                 break;
2244         default:
2245                 dev_warn(fman->dev, "%s: Illegal MAC Id [%d]\n",
2246                          __func__, mac_id);
2247                 return -EINVAL;
2248         }
2249
2250         /* reset */
2251         iowrite32be(msk, &fpm_rg->fm_rstc);
2252         while ((ioread32be(&fpm_rg->fm_rstc) & msk) && --timeout)
2253                 udelay(10);
2254
2255         if (!timeout)
2256                 return -EIO;
2257
2258         return 0;
2259 }
2260 EXPORT_SYMBOL(fman_reset_mac);
2261
2262 /**
2263  * fman_set_mac_max_frame
2264  * @fman:       A Pointer to FMan device
2265  * @mac_id:     MAC id
2266  * @mfl:        Maximum frame length
2267  *
2268  * Set maximum frame length of specific MAC in FMan driver
2269  *
2270  * Return: 0 on success; Error code otherwise.
2271  */
2272 int fman_set_mac_max_frame(struct fman *fman, u8 mac_id, u16 mfl)
2273 {
2274         /* if port is already initialized, check that MaxFrameLength is smaller
2275          * or equal to the port's max
2276          */
2277         if ((!fman->state->port_mfl[mac_id]) ||
2278             (mfl <= fman->state->port_mfl[mac_id])) {
2279                 fman->state->mac_mfl[mac_id] = mfl;
2280         } else {
2281                 dev_warn(fman->dev, "%s: MAC max_frame_length is larger than Port max_frame_length\n",
2282                          __func__);
2283                 return -EINVAL;
2284         }
2285         return 0;
2286 }
2287 EXPORT_SYMBOL(fman_set_mac_max_frame);
2288
2289 /**
2290  * fman_get_clock_freq
2291  * @fman:       A Pointer to FMan device
2292  *
2293  * Get FMan clock frequency
2294  *
2295  * Return: FMan clock frequency
2296  */
2297 u16 fman_get_clock_freq(struct fman *fman)
2298 {
2299         return fman->state->fm_clk_freq;
2300 }
2301
2302 /**
2303  * fman_get_bmi_max_fifo_size
2304  * @fman:       A Pointer to FMan device
2305  *
2306  * Get FMan maximum FIFO size
2307  *
2308  * Return: FMan Maximum FIFO size
2309  */
2310 u32 fman_get_bmi_max_fifo_size(struct fman *fman)
2311 {
2312         return fman->state->bmi_max_fifo_size;
2313 }
2314 EXPORT_SYMBOL(fman_get_bmi_max_fifo_size);
2315
2316 /**
2317  * fman_get_revision
2318  * @fman:               - Pointer to the FMan module
2319  * @rev_info:           - A structure of revision information parameters.
2320  *
2321  * Returns the FM revision
2322  *
2323  * Allowed only following fman_init().
2324  *
2325  * Return: 0 on success; Error code otherwise.
2326  */
2327 void fman_get_revision(struct fman *fman, struct fman_rev_info *rev_info)
2328 {
2329         u32 tmp;
2330
2331         tmp = ioread32be(&fman->fpm_regs->fm_ip_rev_1);
2332         rev_info->major = (u8)((tmp & FPM_REV1_MAJOR_MASK) >>
2333                                 FPM_REV1_MAJOR_SHIFT);
2334         rev_info->minor = tmp & FPM_REV1_MINOR_MASK;
2335 }
2336 EXPORT_SYMBOL(fman_get_revision);
2337
2338 /**
2339  * fman_get_qman_channel_id
2340  * @fman:       A Pointer to FMan device
2341  * @port_id:    Port id
2342  *
2343  * Get QMan channel ID associated to the Port id
2344  *
2345  * Return: QMan channel ID
2346  */
2347 u32 fman_get_qman_channel_id(struct fman *fman, u32 port_id)
2348 {
2349         int i;
2350
2351         if (fman->state->rev_info.major >= 6) {
2352                 static const u32 port_ids[] = {
2353                         0x30, 0x31, 0x28, 0x29, 0x2a, 0x2b,
2354                         0x2c, 0x2d, 0x2, 0x3, 0x4, 0x5, 0x7, 0x7
2355                 };
2356
2357                 for (i = 0; i < fman->state->num_of_qman_channels; i++) {
2358                         if (port_ids[i] == port_id)
2359                                 break;
2360                 }
2361         } else {
2362                 static const u32 port_ids[] = {
2363                         0x30, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x1,
2364                         0x2, 0x3, 0x4, 0x5, 0x7, 0x7
2365                 };
2366
2367                 for (i = 0; i < fman->state->num_of_qman_channels; i++) {
2368                         if (port_ids[i] == port_id)
2369                                 break;
2370                 }
2371         }
2372
2373         if (i == fman->state->num_of_qman_channels)
2374                 return 0;
2375
2376         return fman->state->qman_channel_base + i;
2377 }
2378 EXPORT_SYMBOL(fman_get_qman_channel_id);
2379
2380 /**
2381  * fman_get_mem_region
2382  * @fman:       A Pointer to FMan device
2383  *
2384  * Get FMan memory region
2385  *
2386  * Return: A structure with FMan memory region information
2387  */
2388 struct resource *fman_get_mem_region(struct fman *fman)
2389 {
2390         return fman->state->res;
2391 }
2392 EXPORT_SYMBOL(fman_get_mem_region);
2393
2394 /* Bootargs defines */
2395 /* Extra headroom for RX buffers - Default, min and max */
2396 #define FSL_FM_RX_EXTRA_HEADROOM        64
2397 #define FSL_FM_RX_EXTRA_HEADROOM_MIN    16
2398 #define FSL_FM_RX_EXTRA_HEADROOM_MAX    384
2399
2400 /* Maximum frame length */
2401 #define FSL_FM_MAX_FRAME_SIZE                   1522
2402 #define FSL_FM_MAX_POSSIBLE_FRAME_SIZE          9600
2403 #define FSL_FM_MIN_POSSIBLE_FRAME_SIZE          64
2404
2405 /* Extra headroom for Rx buffers.
2406  * FMan is instructed to allocate, on the Rx path, this amount of
2407  * space at the beginning of a data buffer, beside the DPA private
2408  * data area and the IC fields.
2409  * Does not impact Tx buffer layout.
2410  * Configurable from bootargs. 64 by default, it's needed on
2411  * particular forwarding scenarios that add extra headers to the
2412  * forwarded frame.
2413  */
2414 static int fsl_fm_rx_extra_headroom = FSL_FM_RX_EXTRA_HEADROOM;
2415 module_param(fsl_fm_rx_extra_headroom, int, 0);
2416 MODULE_PARM_DESC(fsl_fm_rx_extra_headroom, "Extra headroom for Rx buffers");
2417
2418 /* Max frame size, across all interfaces.
2419  * Configurable from bootargs, to avoid allocating oversized (socket)
2420  * buffers when not using jumbo frames.
2421  * Must be large enough to accommodate the network MTU, but small enough
2422  * to avoid wasting skb memory.
2423  */
2424 static int fsl_fm_max_frm = FSL_FM_MAX_FRAME_SIZE;
2425 module_param(fsl_fm_max_frm, int, 0);
2426 MODULE_PARM_DESC(fsl_fm_max_frm, "Maximum frame size, across all interfaces");
2427
2428 /**
2429  * fman_get_max_frm
2430  *
2431  * Return: Max frame length configured in the FM driver
2432  */
2433 u16 fman_get_max_frm(void)
2434 {
2435         static bool fm_check_mfl;
2436
2437         if (!fm_check_mfl) {
2438                 if (fsl_fm_max_frm > FSL_FM_MAX_POSSIBLE_FRAME_SIZE ||
2439                     fsl_fm_max_frm < FSL_FM_MIN_POSSIBLE_FRAME_SIZE) {
2440                         pr_warn("Invalid fsl_fm_max_frm value (%d) in bootargs, valid range is %d-%d. Falling back to the default (%d)\n",
2441                                 fsl_fm_max_frm,
2442                                 FSL_FM_MIN_POSSIBLE_FRAME_SIZE,
2443                                 FSL_FM_MAX_POSSIBLE_FRAME_SIZE,
2444                                 FSL_FM_MAX_FRAME_SIZE);
2445                         fsl_fm_max_frm = FSL_FM_MAX_FRAME_SIZE;
2446                 }
2447                 fm_check_mfl = true;
2448         }
2449
2450         return fsl_fm_max_frm;
2451 }
2452 EXPORT_SYMBOL(fman_get_max_frm);
2453
2454 /**
2455  * fman_get_rx_extra_headroom
2456  *
2457  * Return: Extra headroom size configured in the FM driver
2458  */
2459 int fman_get_rx_extra_headroom(void)
2460 {
2461         static bool fm_check_rx_extra_headroom;
2462
2463         if (!fm_check_rx_extra_headroom) {
2464                 if (fsl_fm_rx_extra_headroom > FSL_FM_RX_EXTRA_HEADROOM_MAX ||
2465                     fsl_fm_rx_extra_headroom < FSL_FM_RX_EXTRA_HEADROOM_MIN) {
2466                         pr_warn("Invalid fsl_fm_rx_extra_headroom value (%d) in bootargs, valid range is %d-%d. Falling back to the default (%d)\n",
2467                                 fsl_fm_rx_extra_headroom,
2468                                 FSL_FM_RX_EXTRA_HEADROOM_MIN,
2469                                 FSL_FM_RX_EXTRA_HEADROOM_MAX,
2470                                 FSL_FM_RX_EXTRA_HEADROOM);
2471                         fsl_fm_rx_extra_headroom = FSL_FM_RX_EXTRA_HEADROOM;
2472                 }
2473
2474                 fm_check_rx_extra_headroom = true;
2475                 fsl_fm_rx_extra_headroom = ALIGN(fsl_fm_rx_extra_headroom, 16);
2476         }
2477
2478         return fsl_fm_rx_extra_headroom;
2479 }
2480 EXPORT_SYMBOL(fman_get_rx_extra_headroom);
2481
2482 /**
2483  * fman_bind
2484  * @fm_dev:     FMan OF device pointer
2485  *
2486  * Bind to a specific FMan device.
2487  *
2488  * Allowed only after the port was created.
2489  *
2490  * Return: A pointer to the FMan device
2491  */
2492 struct fman *fman_bind(struct device *fm_dev)
2493 {
2494         return (struct fman *)(dev_get_drvdata(get_device(fm_dev)));
2495 }
2496 EXPORT_SYMBOL(fman_bind);
2497
2498 #ifdef CONFIG_DPAA_ERRATUM_A050385
2499 bool fman_has_errata_a050385(void)
2500 {
2501         return fman_has_err_a050385;
2502 }
2503 EXPORT_SYMBOL(fman_has_errata_a050385);
2504 #endif
2505
2506 static irqreturn_t fman_err_irq(int irq, void *handle)
2507 {
2508         struct fman *fman = (struct fman *)handle;
2509         u32 pending;
2510         struct fman_fpm_regs __iomem *fpm_rg;
2511         irqreturn_t single_ret, ret = IRQ_NONE;
2512
2513         if (!is_init_done(fman->cfg))
2514                 return IRQ_NONE;
2515
2516         fpm_rg = fman->fpm_regs;
2517
2518         /* error interrupts */
2519         pending = ioread32be(&fpm_rg->fm_epi);
2520         if (!pending)
2521                 return IRQ_NONE;
2522
2523         if (pending & ERR_INTR_EN_BMI) {
2524                 single_ret = bmi_err_event(fman);
2525                 if (single_ret == IRQ_HANDLED)
2526                         ret = IRQ_HANDLED;
2527         }
2528         if (pending & ERR_INTR_EN_QMI) {
2529                 single_ret = qmi_err_event(fman);
2530                 if (single_ret == IRQ_HANDLED)
2531                         ret = IRQ_HANDLED;
2532         }
2533         if (pending & ERR_INTR_EN_FPM) {
2534                 single_ret = fpm_err_event(fman);
2535                 if (single_ret == IRQ_HANDLED)
2536                         ret = IRQ_HANDLED;
2537         }
2538         if (pending & ERR_INTR_EN_DMA) {
2539                 single_ret = dma_err_event(fman);
2540                 if (single_ret == IRQ_HANDLED)
2541                         ret = IRQ_HANDLED;
2542         }
2543         if (pending & ERR_INTR_EN_MURAM) {
2544                 single_ret = muram_err_intr(fman);
2545                 if (single_ret == IRQ_HANDLED)
2546                         ret = IRQ_HANDLED;
2547         }
2548
2549         /* MAC error interrupts */
2550         if (pending & ERR_INTR_EN_MAC0) {
2551                 single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 0);
2552                 if (single_ret == IRQ_HANDLED)
2553                         ret = IRQ_HANDLED;
2554         }
2555         if (pending & ERR_INTR_EN_MAC1) {
2556                 single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 1);
2557                 if (single_ret == IRQ_HANDLED)
2558                         ret = IRQ_HANDLED;
2559         }
2560         if (pending & ERR_INTR_EN_MAC2) {
2561                 single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 2);
2562                 if (single_ret == IRQ_HANDLED)
2563                         ret = IRQ_HANDLED;
2564         }
2565         if (pending & ERR_INTR_EN_MAC3) {
2566                 single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 3);
2567                 if (single_ret == IRQ_HANDLED)
2568                         ret = IRQ_HANDLED;
2569         }
2570         if (pending & ERR_INTR_EN_MAC4) {
2571                 single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 4);
2572                 if (single_ret == IRQ_HANDLED)
2573                         ret = IRQ_HANDLED;
2574         }
2575         if (pending & ERR_INTR_EN_MAC5) {
2576                 single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 5);
2577                 if (single_ret == IRQ_HANDLED)
2578                         ret = IRQ_HANDLED;
2579         }
2580         if (pending & ERR_INTR_EN_MAC6) {
2581                 single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 6);
2582                 if (single_ret == IRQ_HANDLED)
2583                         ret = IRQ_HANDLED;
2584         }
2585         if (pending & ERR_INTR_EN_MAC7) {
2586                 single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 7);
2587                 if (single_ret == IRQ_HANDLED)
2588                         ret = IRQ_HANDLED;
2589         }
2590         if (pending & ERR_INTR_EN_MAC8) {
2591                 single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 8);
2592                 if (single_ret == IRQ_HANDLED)
2593                         ret = IRQ_HANDLED;
2594         }
2595         if (pending & ERR_INTR_EN_MAC9) {
2596                 single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 9);
2597                 if (single_ret == IRQ_HANDLED)
2598                         ret = IRQ_HANDLED;
2599         }
2600
2601         return ret;
2602 }
2603
2604 static irqreturn_t fman_irq(int irq, void *handle)
2605 {
2606         struct fman *fman = (struct fman *)handle;
2607         u32 pending;
2608         struct fman_fpm_regs __iomem *fpm_rg;
2609         irqreturn_t single_ret, ret = IRQ_NONE;
2610
2611         if (!is_init_done(fman->cfg))
2612                 return IRQ_NONE;
2613
2614         fpm_rg = fman->fpm_regs;
2615
2616         /* normal interrupts */
2617         pending = ioread32be(&fpm_rg->fm_npi);
2618         if (!pending)
2619                 return IRQ_NONE;
2620
2621         if (pending & INTR_EN_QMI) {
2622                 single_ret = qmi_event(fman);
2623                 if (single_ret == IRQ_HANDLED)
2624                         ret = IRQ_HANDLED;
2625         }
2626
2627         /* MAC interrupts */
2628         if (pending & INTR_EN_MAC0) {
2629                 single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 0);
2630                 if (single_ret == IRQ_HANDLED)
2631                         ret = IRQ_HANDLED;
2632         }
2633         if (pending & INTR_EN_MAC1) {
2634                 single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 1);
2635                 if (single_ret == IRQ_HANDLED)
2636                         ret = IRQ_HANDLED;
2637         }
2638         if (pending & INTR_EN_MAC2) {
2639                 single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 2);
2640                 if (single_ret == IRQ_HANDLED)
2641                         ret = IRQ_HANDLED;
2642         }
2643         if (pending & INTR_EN_MAC3) {
2644                 single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 3);
2645                 if (single_ret == IRQ_HANDLED)
2646                         ret = IRQ_HANDLED;
2647         }
2648         if (pending & INTR_EN_MAC4) {
2649                 single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 4);
2650                 if (single_ret == IRQ_HANDLED)
2651                         ret = IRQ_HANDLED;
2652         }
2653         if (pending & INTR_EN_MAC5) {
2654                 single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 5);
2655                 if (single_ret == IRQ_HANDLED)
2656                         ret = IRQ_HANDLED;
2657         }
2658         if (pending & INTR_EN_MAC6) {
2659                 single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 6);
2660                 if (single_ret == IRQ_HANDLED)
2661                         ret = IRQ_HANDLED;
2662         }
2663         if (pending & INTR_EN_MAC7) {
2664                 single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 7);
2665                 if (single_ret == IRQ_HANDLED)
2666                         ret = IRQ_HANDLED;
2667         }
2668         if (pending & INTR_EN_MAC8) {
2669                 single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 8);
2670                 if (single_ret == IRQ_HANDLED)
2671                         ret = IRQ_HANDLED;
2672         }
2673         if (pending & INTR_EN_MAC9) {
2674                 single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 9);
2675                 if (single_ret == IRQ_HANDLED)
2676                         ret = IRQ_HANDLED;
2677         }
2678
2679         return ret;
2680 }
2681
2682 static const struct of_device_id fman_muram_match[] = {
2683         {
2684                 .compatible = "fsl,fman-muram"},
2685         {}
2686 };
2687 MODULE_DEVICE_TABLE(of, fman_muram_match);
2688
2689 static struct fman *read_dts_node(struct platform_device *of_dev)
2690 {
2691         struct fman *fman;
2692         struct device_node *fm_node, *muram_node;
2693         void __iomem *base_addr;
2694         struct resource *res;
2695         u32 val, range[2];
2696         int err, irq;
2697         struct clk *clk;
2698         u32 clk_rate;
2699
2700         fman = kzalloc(sizeof(*fman), GFP_KERNEL);
2701         if (!fman)
2702                 return ERR_PTR(-ENOMEM);
2703
2704         fm_node = of_node_get(of_dev->dev.of_node);
2705
2706         err = of_property_read_u32(fm_node, "cell-index", &val);
2707         if (err) {
2708                 dev_err(&of_dev->dev, "%s: failed to read cell-index for %pOF\n",
2709                         __func__, fm_node);
2710                 goto fman_node_put;
2711         }
2712         fman->dts_params.id = (u8)val;
2713
2714         /* Get the FM interrupt */
2715         err = platform_get_irq(of_dev, 0);
2716         if (err < 0)
2717                 goto fman_node_put;
2718         irq = err;
2719
2720         /* Get the FM error interrupt */
2721         err = platform_get_irq(of_dev, 1);
2722         if (err < 0)
2723                 goto fman_node_put;
2724         fman->dts_params.err_irq = err;
2725
2726         clk = of_clk_get(fm_node, 0);
2727         if (IS_ERR(clk)) {
2728                 err = PTR_ERR(clk);
2729                 dev_err(&of_dev->dev, "%s: Failed to get FM%d clock structure\n",
2730                         __func__, fman->dts_params.id);
2731                 goto fman_node_put;
2732         }
2733
2734         clk_rate = clk_get_rate(clk);
2735         if (!clk_rate) {
2736                 err = -EINVAL;
2737                 dev_err(&of_dev->dev, "%s: Failed to determine FM%d clock rate\n",
2738                         __func__, fman->dts_params.id);
2739                 goto fman_node_put;
2740         }
2741         /* Rounding to MHz */
2742         fman->dts_params.clk_freq = DIV_ROUND_UP(clk_rate, 1000000);
2743
2744         err = of_property_read_u32_array(fm_node, "fsl,qman-channel-range",
2745                                          &range[0], 2);
2746         if (err) {
2747                 dev_err(&of_dev->dev, "%s: failed to read fsl,qman-channel-range for %pOF\n",
2748                         __func__, fm_node);
2749                 goto fman_node_put;
2750         }
2751         fman->dts_params.qman_channel_base = range[0];
2752         fman->dts_params.num_of_qman_channels = range[1];
2753
2754         /* Get the MURAM base address and size */
2755         muram_node = of_find_matching_node(fm_node, fman_muram_match);
2756         if (!muram_node) {
2757                 err = -EINVAL;
2758                 dev_err(&of_dev->dev, "%s: could not find MURAM node\n",
2759                         __func__);
2760                 goto fman_free;
2761         }
2762
2763         err = of_address_to_resource(muram_node, 0,
2764                                      &fman->dts_params.muram_res);
2765         if (err) {
2766                 of_node_put(muram_node);
2767                 dev_err(&of_dev->dev, "%s: of_address_to_resource() = %d\n",
2768                         __func__, err);
2769                 goto fman_free;
2770         }
2771
2772         of_node_put(muram_node);
2773
2774         err = devm_request_irq(&of_dev->dev, irq, fman_irq, IRQF_SHARED,
2775                                "fman", fman);
2776         if (err < 0) {
2777                 dev_err(&of_dev->dev, "%s: irq %d allocation failed (error = %d)\n",
2778                         __func__, irq, err);
2779                 goto fman_free;
2780         }
2781
2782         if (fman->dts_params.err_irq != 0) {
2783                 err = devm_request_irq(&of_dev->dev, fman->dts_params.err_irq,
2784                                        fman_err_irq, IRQF_SHARED,
2785                                        "fman-err", fman);
2786                 if (err < 0) {
2787                         dev_err(&of_dev->dev, "%s: irq %d allocation failed (error = %d)\n",
2788                                 __func__, fman->dts_params.err_irq, err);
2789                         goto fman_free;
2790                 }
2791         }
2792
2793         base_addr = devm_platform_get_and_ioremap_resource(of_dev, 0, &res);
2794         if (IS_ERR(base_addr)) {
2795                 err = PTR_ERR(base_addr);
2796                 dev_err(&of_dev->dev, "%s: devm_ioremap() failed\n", __func__);
2797                 goto fman_free;
2798         }
2799
2800         fman->dts_params.base_addr = base_addr;
2801         fman->dts_params.res = res;
2802
2803         fman->dev = &of_dev->dev;
2804
2805         err = of_platform_populate(fm_node, NULL, NULL, &of_dev->dev);
2806         if (err) {
2807                 dev_err(&of_dev->dev, "%s: of_platform_populate() failed\n",
2808                         __func__);
2809                 goto fman_free;
2810         }
2811
2812 #ifdef CONFIG_DPAA_ERRATUM_A050385
2813         fman_has_err_a050385 =
2814                 of_property_read_bool(fm_node, "fsl,erratum-a050385");
2815 #endif
2816
2817         return fman;
2818
2819 fman_node_put:
2820         of_node_put(fm_node);
2821 fman_free:
2822         kfree(fman);
2823         return ERR_PTR(err);
2824 }
2825
2826 static int fman_probe(struct platform_device *of_dev)
2827 {
2828         struct fman *fman;
2829         struct device *dev;
2830         int err;
2831
2832         dev = &of_dev->dev;
2833
2834         fman = read_dts_node(of_dev);
2835         if (IS_ERR(fman))
2836                 return PTR_ERR(fman);
2837
2838         err = fman_config(fman);
2839         if (err) {
2840                 dev_err(dev, "%s: FMan config failed\n", __func__);
2841                 return -EINVAL;
2842         }
2843
2844         if (fman_init(fman) != 0) {
2845                 dev_err(dev, "%s: FMan init failed\n", __func__);
2846                 return -EINVAL;
2847         }
2848
2849         if (fman->dts_params.err_irq == 0) {
2850                 fman_set_exception(fman, FMAN_EX_DMA_BUS_ERROR, false);
2851                 fman_set_exception(fman, FMAN_EX_DMA_READ_ECC, false);
2852                 fman_set_exception(fman, FMAN_EX_DMA_SYSTEM_WRITE_ECC, false);
2853                 fman_set_exception(fman, FMAN_EX_DMA_FM_WRITE_ECC, false);
2854                 fman_set_exception(fman, FMAN_EX_DMA_SINGLE_PORT_ECC, false);
2855                 fman_set_exception(fman, FMAN_EX_FPM_STALL_ON_TASKS, false);
2856                 fman_set_exception(fman, FMAN_EX_FPM_SINGLE_ECC, false);
2857                 fman_set_exception(fman, FMAN_EX_FPM_DOUBLE_ECC, false);
2858                 fman_set_exception(fman, FMAN_EX_QMI_SINGLE_ECC, false);
2859                 fman_set_exception(fman, FMAN_EX_QMI_DOUBLE_ECC, false);
2860                 fman_set_exception(fman,
2861                                    FMAN_EX_QMI_DEQ_FROM_UNKNOWN_PORTID, false);
2862                 fman_set_exception(fman, FMAN_EX_BMI_LIST_RAM_ECC, false);
2863                 fman_set_exception(fman, FMAN_EX_BMI_STORAGE_PROFILE_ECC,
2864                                    false);
2865                 fman_set_exception(fman, FMAN_EX_BMI_STATISTICS_RAM_ECC, false);
2866                 fman_set_exception(fman, FMAN_EX_BMI_DISPATCH_RAM_ECC, false);
2867         }
2868
2869         dev_set_drvdata(dev, fman);
2870
2871         dev_dbg(dev, "FMan%d probed\n", fman->dts_params.id);
2872
2873         return 0;
2874 }
2875
2876 static const struct of_device_id fman_match[] = {
2877         {
2878                 .compatible = "fsl,fman"},
2879         {}
2880 };
2881
2882 MODULE_DEVICE_TABLE(of, fman_match);
2883
2884 static struct platform_driver fman_driver = {
2885         .driver = {
2886                 .name = "fsl-fman",
2887                 .of_match_table = fman_match,
2888         },
2889         .probe = fman_probe,
2890 };
2891
2892 static int __init fman_load(void)
2893 {
2894         int err;
2895
2896         pr_debug("FSL DPAA FMan driver\n");
2897
2898         err = platform_driver_register(&fman_driver);
2899         if (err < 0)
2900                 pr_err("Error, platform_driver_register() = %d\n", err);
2901
2902         return err;
2903 }
2904 module_init(fman_load);
2905
2906 static void __exit fman_unload(void)
2907 {
2908         platform_driver_unregister(&fman_driver);
2909 }
2910 module_exit(fman_unload);
2911
2912 MODULE_LICENSE("Dual BSD/GPL");
2913 MODULE_DESCRIPTION("Freescale DPAA Frame Manager driver");