drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.c

   1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
   2 /* Copyright(c) 2020 Intel Corporation */
   3 #include <linux/iopoll.h>
   4 #include <asm/div64.h>
   5 #include "adf_accel_devices.h"
   6 #include "adf_cfg_services.h"
   7 #include "adf_common_drv.h"
   8 #include "adf_fw_config.h"
   9 #include "adf_gen4_hw_data.h"
  10 #include "adf_gen4_pm.h"
  11
  12 u32 adf_gen4_get_accel_mask(struct adf_hw_device_data *self)
  13 {
  14         return ADF_GEN4_ACCELERATORS_MASK;
  15 }
  16 EXPORT_SYMBOL_GPL(adf_gen4_get_accel_mask);
  17
  18 u32 adf_gen4_get_num_accels(struct adf_hw_device_data *self)
  19 {
  20         return ADF_GEN4_MAX_ACCELERATORS;
  21 }
  22 EXPORT_SYMBOL_GPL(adf_gen4_get_num_accels);
  23
  24 u32 adf_gen4_get_num_aes(struct adf_hw_device_data *self)
  25 {
  26         if (!self || !self->ae_mask)
  27                 return 0;
  28
  29         return hweight32(self->ae_mask);
  30 }
  31 EXPORT_SYMBOL_GPL(adf_gen4_get_num_aes);
  32
  33 u32 adf_gen4_get_misc_bar_id(struct adf_hw_device_data *self)
  34 {
  35         return ADF_GEN4_PMISC_BAR;
  36 }
  37 EXPORT_SYMBOL_GPL(adf_gen4_get_misc_bar_id);
  38
  39 u32 adf_gen4_get_etr_bar_id(struct adf_hw_device_data *self)
  40 {
  41         return ADF_GEN4_ETR_BAR;
  42 }
  43 EXPORT_SYMBOL_GPL(adf_gen4_get_etr_bar_id);
  44
  45 u32 adf_gen4_get_sram_bar_id(struct adf_hw_device_data *self)
  46 {
  47         return ADF_GEN4_SRAM_BAR;
  48 }
  49 EXPORT_SYMBOL_GPL(adf_gen4_get_sram_bar_id);
  50
  51 enum dev_sku_info adf_gen4_get_sku(struct adf_hw_device_data *self)
  52 {
  53         return DEV_SKU_1;
  54 }
  55 EXPORT_SYMBOL_GPL(adf_gen4_get_sku);
  56
  57 void adf_gen4_get_arb_info(struct arb_info *arb_info)
  58 {
  59         arb_info->arb_cfg = ADF_GEN4_ARB_CONFIG;
  60         arb_info->arb_offset = ADF_GEN4_ARB_OFFSET;
  61         arb_info->wt2sam_offset = ADF_GEN4_ARB_WRK_2_SER_MAP_OFFSET;
  62 }
  63 EXPORT_SYMBOL_GPL(adf_gen4_get_arb_info);
  64
  65 void adf_gen4_get_admin_info(struct admin_info *admin_csrs_info)
  66 {
  67         admin_csrs_info->mailbox_offset = ADF_GEN4_MAILBOX_BASE_OFFSET;
  68         admin_csrs_info->admin_msg_ur = ADF_GEN4_ADMINMSGUR_OFFSET;
  69         admin_csrs_info->admin_msg_lr = ADF_GEN4_ADMINMSGLR_OFFSET;
  70 }
  71 EXPORT_SYMBOL_GPL(adf_gen4_get_admin_info);
  72
  73 u32 adf_gen4_get_heartbeat_clock(struct adf_hw_device_data *self)
  74 {
  75         /*
  76          * GEN4 uses KPT counter for HB
  77          */
  78         return ADF_GEN4_KPT_COUNTER_FREQ;
  79 }
  80 EXPORT_SYMBOL_GPL(adf_gen4_get_heartbeat_clock);
  81
  82 void adf_gen4_enable_error_correction(struct adf_accel_dev *accel_dev)
  83 {
  84         struct adf_bar *misc_bar = &GET_BARS(accel_dev)[ADF_GEN4_PMISC_BAR];
  85         void __iomem *csr = misc_bar->virt_addr;
  86
  87         /* Enable all in errsou3 except VFLR notification on host */
  88         ADF_CSR_WR(csr, ADF_GEN4_ERRMSK3, ADF_GEN4_VFLNOTIFY);
  89 }
  90 EXPORT_SYMBOL_GPL(adf_gen4_enable_error_correction);
  91
  92 void adf_gen4_enable_ints(struct adf_accel_dev *accel_dev)
  93 {
  94         void __iomem *addr;
  95
  96         addr = (&GET_BARS(accel_dev)[ADF_GEN4_PMISC_BAR])->virt_addr;
  97
  98         /* Enable bundle interrupts */
  99         ADF_CSR_WR(addr, ADF_GEN4_SMIAPF_RP_X0_MASK_OFFSET, 0);
 100         ADF_CSR_WR(addr, ADF_GEN4_SMIAPF_RP_X1_MASK_OFFSET, 0);
 101
 102         /* Enable misc interrupts */
 103         ADF_CSR_WR(addr, ADF_GEN4_SMIAPF_MASK_OFFSET, 0);
 104 }
 105 EXPORT_SYMBOL_GPL(adf_gen4_enable_ints);
 106
 107 int adf_gen4_init_device(struct adf_accel_dev *accel_dev)
 108 {
 109         void __iomem *addr;
 110         u32 status;
 111         u32 csr;
 112         int ret;
 113
 114         addr = (&GET_BARS(accel_dev)[ADF_GEN4_PMISC_BAR])->virt_addr;
 115
 116         /* Temporarily mask PM interrupt */
 117         csr = ADF_CSR_RD(addr, ADF_GEN4_ERRMSK2);
 118         csr |= ADF_GEN4_PM_SOU;
 119         ADF_CSR_WR(addr, ADF_GEN4_ERRMSK2, csr);
 120
 121         /* Set DRV_ACTIVE bit to power up the device */
 122         ADF_CSR_WR(addr, ADF_GEN4_PM_INTERRUPT, ADF_GEN4_PM_DRV_ACTIVE);
 123
 124         /* Poll status register to make sure the device is powered up */
 125         ret = read_poll_timeout(ADF_CSR_RD, status,
 126                                 status & ADF_GEN4_PM_INIT_STATE,
 127                                 ADF_GEN4_PM_POLL_DELAY_US,
 128                                 ADF_GEN4_PM_POLL_TIMEOUT_US, true, addr,
 129                                 ADF_GEN4_PM_STATUS);
 130         if (ret)
 131                 dev_err(&GET_DEV(accel_dev), "Failed to power up the device\n");
 132
 133         return ret;
 134 }
 135 EXPORT_SYMBOL_GPL(adf_gen4_init_device);
 136
 137 static inline void adf_gen4_unpack_ssm_wdtimer(u64 value, u32 *upper,
 138                                                u32 *lower)
 139 {
 140         *lower = lower_32_bits(value);
 141         *upper = upper_32_bits(value);
 142 }
 143
 144 void adf_gen4_set_ssm_wdtimer(struct adf_accel_dev *accel_dev)
 145 {
 146         void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
 147         u64 timer_val_pke = ADF_SSM_WDT_PKE_DEFAULT_VALUE;
 148         u64 timer_val = ADF_SSM_WDT_DEFAULT_VALUE;
 149         u32 ssm_wdt_pke_high = 0;
 150         u32 ssm_wdt_pke_low = 0;
 151         u32 ssm_wdt_high = 0;
 152         u32 ssm_wdt_low = 0;
 153
 154         /* Convert 64bit WDT timer value into 32bit values for
 155          * mmio write to 32bit CSRs.
 156          */
 157         adf_gen4_unpack_ssm_wdtimer(timer_val, &ssm_wdt_high, &ssm_wdt_low);
 158         adf_gen4_unpack_ssm_wdtimer(timer_val_pke, &ssm_wdt_pke_high,
 159                                     &ssm_wdt_pke_low);
 160
 161         /* Enable WDT for sym and dc */
 162         ADF_CSR_WR(pmisc_addr, ADF_SSMWDTL_OFFSET, ssm_wdt_low);
 163         ADF_CSR_WR(pmisc_addr, ADF_SSMWDTH_OFFSET, ssm_wdt_high);
 164         /* Enable WDT for pke */
 165         ADF_CSR_WR(pmisc_addr, ADF_SSMWDTPKEL_OFFSET, ssm_wdt_pke_low);
 166         ADF_CSR_WR(pmisc_addr, ADF_SSMWDTPKEH_OFFSET, ssm_wdt_pke_high);
 167 }
 168 EXPORT_SYMBOL_GPL(adf_gen4_set_ssm_wdtimer);
 169
 170 /*
 171  * The vector routing table is used to select the MSI-X entry to use for each
 172  * interrupt source.
 173  * The first ADF_GEN4_ETR_MAX_BANKS entries correspond to ring interrupts.
 174  * The final entry corresponds to VF2PF or error interrupts.
 175  * This vector table could be used to configure one MSI-X entry to be shared
 176  * between multiple interrupt sources.
 177  *
 178  * The default routing is set to have a one to one correspondence between the
 179  * interrupt source and the MSI-X entry used.
 180  */
 181 void adf_gen4_set_msix_default_rttable(struct adf_accel_dev *accel_dev)
 182 {
 183         void __iomem *csr;
 184         int i;
 185
 186         csr = (&GET_BARS(accel_dev)[ADF_GEN4_PMISC_BAR])->virt_addr;
 187         for (i = 0; i <= ADF_GEN4_ETR_MAX_BANKS; i++)
 188                 ADF_CSR_WR(csr, ADF_GEN4_MSIX_RTTABLE_OFFSET(i), i);
 189 }
 190 EXPORT_SYMBOL_GPL(adf_gen4_set_msix_default_rttable);
 191
 192 int adf_pfvf_comms_disabled(struct adf_accel_dev *accel_dev)
 193 {
 194         return 0;
 195 }
 196 EXPORT_SYMBOL_GPL(adf_pfvf_comms_disabled);
 197
 198 static int reset_ring_pair(void __iomem *csr, u32 bank_number)
 199 {
 200         u32 status;
 201         int ret;
 202
 203         /* Write rpresetctl register BIT(0) as 1
 204          * Since rpresetctl registers have no RW fields, no need to preserve
 205          * values for other bits. Just write directly.
 206          */
 207         ADF_CSR_WR(csr, ADF_WQM_CSR_RPRESETCTL(bank_number),
 208                    ADF_WQM_CSR_RPRESETCTL_RESET);
 209
 210         /* Read rpresetsts register and wait for rp reset to complete */
 211         ret = read_poll_timeout(ADF_CSR_RD, status,
 212                                 status & ADF_WQM_CSR_RPRESETSTS_STATUS,
 213                                 ADF_RPRESET_POLL_DELAY_US,
 214                                 ADF_RPRESET_POLL_TIMEOUT_US, true,
 215                                 csr, ADF_WQM_CSR_RPRESETSTS(bank_number));
 216         if (!ret) {
 217                 /* When rp reset is done, clear rpresetsts */
 218                 ADF_CSR_WR(csr, ADF_WQM_CSR_RPRESETSTS(bank_number),
 219                            ADF_WQM_CSR_RPRESETSTS_STATUS);
 220         }
 221
 222         return ret;
 223 }
 224
 225 int adf_gen4_ring_pair_reset(struct adf_accel_dev *accel_dev, u32 bank_number)
 226 {
 227         struct adf_hw_device_data *hw_data = accel_dev->hw_device;
 228         void __iomem *csr = adf_get_etr_base(accel_dev);
 229         int ret;
 230
 231         if (bank_number >= hw_data->num_banks)
 232                 return -EINVAL;
 233
 234         dev_dbg(&GET_DEV(accel_dev),
 235                 "ring pair reset for bank:%d\n", bank_number);
 236
 237         ret = reset_ring_pair(csr, bank_number);
 238         if (ret)
 239                 dev_err(&GET_DEV(accel_dev),
 240                         "ring pair reset failed (timeout)\n");
 241         else
 242                 dev_dbg(&GET_DEV(accel_dev), "ring pair reset successful\n");
 243
 244         return ret;
 245 }
 246 EXPORT_SYMBOL_GPL(adf_gen4_ring_pair_reset);
 247
 248 static const u32 thrd_to_arb_map_dcc[] = {
 249         0x00000000, 0x00000000, 0x00000000, 0x00000000,
 250         0x0000FFFF, 0x0000FFFF, 0x0000FFFF, 0x0000FFFF,
 251         0x00000000, 0x00000000, 0x00000000, 0x00000000,
 252         0x00000000, 0x00000000, 0x00000000, 0x00000000,
 253         0x0
 254 };
 255
 256 static const u16 rp_group_to_arb_mask[] = {
 257         [RP_GROUP_0] = 0x5,
 258         [RP_GROUP_1] = 0xA,
 259 };
 260
 261 static bool is_single_service(int service_id)
 262 {
 263         switch (service_id) {
 264         case SVC_DC:
 265         case SVC_SYM:
 266         case SVC_ASYM:
 267                 return true;
 268         case SVC_CY:
 269         case SVC_CY2:
 270         case SVC_DCC:
 271         case SVC_ASYM_DC:
 272         case SVC_DC_ASYM:
 273         case SVC_SYM_DC:
 274         case SVC_DC_SYM:
 275         default:
 276                 return false;
 277         }
 278 }
 279
 280 int adf_gen4_init_thd2arb_map(struct adf_accel_dev *accel_dev)
 281 {
 282         struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev);
 283         u32 *thd2arb_map = hw_data->thd_to_arb_map;
 284         unsigned int ae_cnt, worker_obj_cnt, i, j;
 285         unsigned long ae_mask, thds_mask;
 286         int srv_id, rp_group;
 287         u32 thd2arb_map_base;
 288         u16 arb_mask;
 289
 290         if (!hw_data->get_rp_group || !hw_data->get_ena_thd_mask ||
 291             !hw_data->get_num_aes || !hw_data->uof_get_num_objs ||
 292             !hw_data->uof_get_ae_mask)
 293                 return -EFAULT;
 294
 295         srv_id = adf_get_service_enabled(accel_dev);
 296         if (srv_id < 0)
 297                 return srv_id;
 298
 299         ae_cnt = hw_data->get_num_aes(hw_data);
 300         worker_obj_cnt = hw_data->uof_get_num_objs(accel_dev) -
 301                          ADF_GEN4_ADMIN_ACCELENGINES;
 302
 303         if (srv_id == SVC_DCC) {
 304                 if (ae_cnt > ICP_QAT_HW_AE_DELIMITER)
 305                         return -EINVAL;
 306
 307                 memcpy(thd2arb_map, thrd_to_arb_map_dcc,
 308                        array_size(sizeof(*thd2arb_map), ae_cnt));
 309                 return 0;
 310         }
 311
 312         for (i = 0; i < worker_obj_cnt; i++) {
 313                 ae_mask = hw_data->uof_get_ae_mask(accel_dev, i);
 314                 rp_group = hw_data->get_rp_group(accel_dev, ae_mask);
 315                 thds_mask = hw_data->get_ena_thd_mask(accel_dev, i);
 316                 thd2arb_map_base = 0;
 317
 318                 if (rp_group >= RP_GROUP_COUNT || rp_group < RP_GROUP_0)
 319                         return -EINVAL;
 320
 321                 if (thds_mask == ADF_GEN4_ENA_THD_MASK_ERROR)
 322                         return -EINVAL;
 323
 324                 if (is_single_service(srv_id))
 325                         arb_mask = rp_group_to_arb_mask[RP_GROUP_0] |
 326                                    rp_group_to_arb_mask[RP_GROUP_1];
 327                 else
 328                         arb_mask = rp_group_to_arb_mask[rp_group];
 329
 330                 for_each_set_bit(j, &thds_mask, ADF_NUM_THREADS_PER_AE)
 331                         thd2arb_map_base |= arb_mask << (j * 4);
 332
 333                 for_each_set_bit(j, &ae_mask, ae_cnt)
 334                         thd2arb_map[j] = thd2arb_map_base;
 335         }
 336         return 0;
 337 }
 338 EXPORT_SYMBOL_GPL(adf_gen4_init_thd2arb_map);
 339
 340 u16 adf_gen4_get_ring_to_svc_map(struct adf_accel_dev *accel_dev)
 341 {
 342         struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev);
 343         enum adf_cfg_service_type rps[RP_GROUP_COUNT] = { };
 344         unsigned int ae_mask, start_id, worker_obj_cnt, i;
 345         u16 ring_to_svc_map;
 346         int rp_group;
 347
 348         if (!hw_data->get_rp_group || !hw_data->uof_get_ae_mask ||
 349             !hw_data->uof_get_obj_type || !hw_data->uof_get_num_objs)
 350                 return 0;
 351
 352         /* If dcc, all rings handle compression requests */
 353         if (adf_get_service_enabled(accel_dev) == SVC_DCC) {
 354                 for (i = 0; i < RP_GROUP_COUNT; i++)
 355                         rps[i] = COMP;
 356                 goto set_mask;
 357         }
 358
 359         worker_obj_cnt = hw_data->uof_get_num_objs(accel_dev) -
 360                          ADF_GEN4_ADMIN_ACCELENGINES;
 361         start_id = worker_obj_cnt - RP_GROUP_COUNT;
 362
 363         for (i = start_id; i < worker_obj_cnt; i++) {
 364                 ae_mask = hw_data->uof_get_ae_mask(accel_dev, i);
 365                 rp_group = hw_data->get_rp_group(accel_dev, ae_mask);
 366                 if (rp_group >= RP_GROUP_COUNT || rp_group < RP_GROUP_0)
 367                         return 0;
 368
 369                 switch (hw_data->uof_get_obj_type(accel_dev, i)) {
 370                 case ADF_FW_SYM_OBJ:
 371                         rps[rp_group] = SYM;
 372                         break;
 373                 case ADF_FW_ASYM_OBJ:
 374                         rps[rp_group] = ASYM;
 375                         break;
 376                 case ADF_FW_DC_OBJ:
 377                         rps[rp_group] = COMP;
 378                         break;
 379                 default:
 380                         rps[rp_group] = 0;
 381                         break;
 382                 }
 383         }
 384
 385 set_mask:
 386         ring_to_svc_map = rps[RP_GROUP_0] << ADF_CFG_SERV_RING_PAIR_0_SHIFT |
 387                           rps[RP_GROUP_1] << ADF_CFG_SERV_RING_PAIR_1_SHIFT |
 388                           rps[RP_GROUP_0] << ADF_CFG_SERV_RING_PAIR_2_SHIFT |
 389                           rps[RP_GROUP_1] << ADF_CFG_SERV_RING_PAIR_3_SHIFT;
 390
 391         return ring_to_svc_map;
 392 }
 393 EXPORT_SYMBOL_GPL(adf_gen4_get_ring_to_svc_map);
 394
 395 /*
 396  * adf_gen4_bank_quiesce_coal_timer() - quiesce bank coalesced interrupt timer
 397  * @accel_dev: Pointer to the device structure
 398  * @bank_idx: Offset to the bank within this device
 399  * @timeout_ms: Timeout in milliseconds for the operation
 400  *
 401  * This function tries to quiesce the coalesced interrupt timer of a bank if
 402  * it has been enabled and triggered.
 403  *
 404  * Returns 0 on success, error code otherwise
 405  *
 406  */
 407 int adf_gen4_bank_quiesce_coal_timer(struct adf_accel_dev *accel_dev,
 408                                      u32 bank_idx, int timeout_ms)
 409 {
 410         struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev);
 411         struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev);
 412         void __iomem *csr_misc = adf_get_pmisc_base(accel_dev);
 413         void __iomem *csr_etr = adf_get_etr_base(accel_dev);
 414         u32 int_col_ctl, int_col_mask, int_col_en;
 415         u32 e_stat, intsrc;
 416         u64 wait_us;
 417         int ret;
 418
 419         if (timeout_ms < 0)
 420                 return -EINVAL;
 421
 422         int_col_ctl = csr_ops->read_csr_int_col_ctl(csr_etr, bank_idx);
 423         int_col_mask = csr_ops->get_int_col_ctl_enable_mask();
 424         if (!(int_col_ctl & int_col_mask))
 425                 return 0;
 426
 427         int_col_en = csr_ops->read_csr_int_col_en(csr_etr, bank_idx);
 428         int_col_en &= BIT(ADF_WQM_CSR_RP_IDX_RX);
 429
 430         e_stat = csr_ops->read_csr_e_stat(csr_etr, bank_idx);
 431         if (!(~e_stat & int_col_en))
 432                 return 0;
 433
 434         wait_us = 2 * ((int_col_ctl & ~int_col_mask) << 8) * USEC_PER_SEC;
 435         do_div(wait_us, hw_data->clock_frequency);
 436         wait_us = min(wait_us, (u64)timeout_ms * USEC_PER_MSEC);
 437         dev_dbg(&GET_DEV(accel_dev),
 438                 "wait for bank %d - coalesced timer expires in %llu us (max=%u ms estat=0x%x intcolen=0x%x)\n",
 439                 bank_idx, wait_us, timeout_ms, e_stat, int_col_en);
 440
 441         ret = read_poll_timeout(ADF_CSR_RD, intsrc, intsrc,
 442                                 ADF_COALESCED_POLL_DELAY_US, wait_us, true,
 443                                 csr_misc, ADF_WQM_CSR_RPINTSOU(bank_idx));
 444         if (ret)
 445                 dev_warn(&GET_DEV(accel_dev),
 446                          "coalesced timer for bank %d expired (%llu us)\n",
 447                          bank_idx, wait_us);
 448
 449         return ret;
 450 }
 451 EXPORT_SYMBOL_GPL(adf_gen4_bank_quiesce_coal_timer);
 452
 453 static int drain_bank(void __iomem *csr, u32 bank_number, int timeout_us)
 454 {
 455         u32 status;
 456
 457         ADF_CSR_WR(csr, ADF_WQM_CSR_RPRESETCTL(bank_number),
 458                    ADF_WQM_CSR_RPRESETCTL_DRAIN);
 459
 460         return read_poll_timeout(ADF_CSR_RD, status,
 461                                 status & ADF_WQM_CSR_RPRESETSTS_STATUS,
 462                                 ADF_RPRESET_POLL_DELAY_US, timeout_us, true,
 463                                 csr, ADF_WQM_CSR_RPRESETSTS(bank_number));
 464 }
 465
 466 void adf_gen4_bank_drain_finish(struct adf_accel_dev *accel_dev,
 467                                 u32 bank_number)
 468 {
 469         void __iomem *csr = adf_get_etr_base(accel_dev);
 470
 471         ADF_CSR_WR(csr, ADF_WQM_CSR_RPRESETSTS(bank_number),
 472                    ADF_WQM_CSR_RPRESETSTS_STATUS);
 473 }
 474
 475 int adf_gen4_bank_drain_start(struct adf_accel_dev *accel_dev,
 476                               u32 bank_number, int timeout_us)
 477 {
 478         void __iomem *csr = adf_get_etr_base(accel_dev);
 479         int ret;
 480
 481         dev_dbg(&GET_DEV(accel_dev), "Drain bank %d\n", bank_number);
 482
 483         ret = drain_bank(csr, bank_number, timeout_us);
 484         if (ret)
 485                 dev_err(&GET_DEV(accel_dev), "Bank drain failed (timeout)\n");
 486         else
 487                 dev_dbg(&GET_DEV(accel_dev), "Bank drain successful\n");
 488
 489         return ret;
 490 }
 491
 492 static void bank_state_save(struct adf_hw_csr_ops *ops, void __iomem *base,
 493                             u32 bank, struct bank_state *state, u32 num_rings)
 494 {
 495         u32 i;
 496
 497         state->ringstat0 = ops->read_csr_stat(base, bank);
 498         state->ringuostat = ops->read_csr_uo_stat(base, bank);
 499         state->ringestat = ops->read_csr_e_stat(base, bank);
 500         state->ringnestat = ops->read_csr_ne_stat(base, bank);
 501         state->ringnfstat = ops->read_csr_nf_stat(base, bank);
 502         state->ringfstat = ops->read_csr_f_stat(base, bank);
 503         state->ringcstat0 = ops->read_csr_c_stat(base, bank);
 504         state->iaintflagen = ops->read_csr_int_en(base, bank);
 505         state->iaintflagreg = ops->read_csr_int_flag(base, bank);
 506         state->iaintflagsrcsel0 = ops->read_csr_int_srcsel(base, bank);
 507         state->iaintcolen = ops->read_csr_int_col_en(base, bank);
 508         state->iaintcolctl = ops->read_csr_int_col_ctl(base, bank);
 509         state->iaintflagandcolen = ops->read_csr_int_flag_and_col(base, bank);
 510         state->ringexpstat = ops->read_csr_exp_stat(base, bank);
 511         state->ringexpintenable = ops->read_csr_exp_int_en(base, bank);
 512         state->ringsrvarben = ops->read_csr_ring_srv_arb_en(base, bank);
 513
 514         for (i = 0; i < num_rings; i++) {
 515                 state->rings[i].head = ops->read_csr_ring_head(base, bank, i);
 516                 state->rings[i].tail = ops->read_csr_ring_tail(base, bank, i);
 517                 state->rings[i].config = ops->read_csr_ring_config(base, bank, i);
 518                 state->rings[i].base = ops->read_csr_ring_base(base, bank, i);
 519         }
 520 }
 521
 522 #define CHECK_STAT(op, expect_val, name, args...) \
 523 ({ \
 524         u32 __expect_val = (expect_val); \
 525         u32 actual_val = op(args); \
 526         (__expect_val == actual_val) ? 0 : \
 527                 (pr_err("QAT: Fail to restore %s register. Expected 0x%x, actual 0x%x\n", \
 528                         name, __expect_val, actual_val), -EINVAL); \
 529 })
 530
 531 static int bank_state_restore(struct adf_hw_csr_ops *ops, void __iomem *base,
 532                               u32 bank, struct bank_state *state, u32 num_rings,
 533                               int tx_rx_gap)
 534 {
 535         u32 val, tmp_val, i;
 536         int ret;
 537
 538         for (i = 0; i < num_rings; i++)
 539                 ops->write_csr_ring_base(base, bank, i, state->rings[i].base);
 540
 541         for (i = 0; i < num_rings; i++)
 542                 ops->write_csr_ring_config(base, bank, i, state->rings[i].config);
 543
 544         for (i = 0; i < num_rings / 2; i++) {
 545                 int tx = i * (tx_rx_gap + 1);
 546                 int rx = tx + tx_rx_gap;
 547
 548                 ops->write_csr_ring_head(base, bank, tx, state->rings[tx].head);
 549                 ops->write_csr_ring_tail(base, bank, tx, state->rings[tx].tail);
 550
 551                 /*
 552                  * The TX ring head needs to be updated again to make sure that
 553                  * the HW will not consider the ring as full when it is empty
 554                  * and the correct state flags are set to match the recovered state.
 555                  */
 556                 if (state->ringestat & BIT(tx)) {
 557                         val = ops->read_csr_int_srcsel(base, bank);
 558                         val |= ADF_RP_INT_SRC_SEL_F_RISE_MASK;
 559                         ops->write_csr_int_srcsel_w_val(base, bank, val);
 560                         ops->write_csr_ring_head(base, bank, tx, state->rings[tx].head);
 561                 }
 562
 563                 ops->write_csr_ring_tail(base, bank, rx, state->rings[rx].tail);
 564                 val = ops->read_csr_int_srcsel(base, bank);
 565                 val |= ADF_RP_INT_SRC_SEL_F_RISE_MASK << ADF_RP_INT_SRC_SEL_RANGE_WIDTH;
 566                 ops->write_csr_int_srcsel_w_val(base, bank, val);
 567
 568                 ops->write_csr_ring_head(base, bank, rx, state->rings[rx].head);
 569                 val = ops->read_csr_int_srcsel(base, bank);
 570                 val |= ADF_RP_INT_SRC_SEL_F_FALL_MASK << ADF_RP_INT_SRC_SEL_RANGE_WIDTH;
 571                 ops->write_csr_int_srcsel_w_val(base, bank, val);
 572
 573                 /*
 574                  * The RX ring tail needs to be updated again to make sure that
 575                  * the HW will not consider the ring as empty when it is full
 576                  * and the correct state flags are set to match the recovered state.
 577                  */
 578                 if (state->ringfstat & BIT(rx))
 579                         ops->write_csr_ring_tail(base, bank, rx, state->rings[rx].tail);
 580         }
 581
 582         ops->write_csr_int_flag_and_col(base, bank, state->iaintflagandcolen);
 583         ops->write_csr_int_en(base, bank, state->iaintflagen);
 584         ops->write_csr_int_col_en(base, bank, state->iaintcolen);
 585         ops->write_csr_int_srcsel_w_val(base, bank, state->iaintflagsrcsel0);
 586         ops->write_csr_exp_int_en(base, bank, state->ringexpintenable);
 587         ops->write_csr_int_col_ctl(base, bank, state->iaintcolctl);
 588         ops->write_csr_ring_srv_arb_en(base, bank, state->ringsrvarben);
 589
 590         /* Check that all ring statuses match the saved state. */
 591         ret = CHECK_STAT(ops->read_csr_stat, state->ringstat0, "ringstat",
 592                          base, bank);
 593         if (ret)
 594                 return ret;
 595
 596         ret = CHECK_STAT(ops->read_csr_e_stat, state->ringestat, "ringestat",
 597                          base, bank);
 598         if (ret)
 599                 return ret;
 600
 601         ret = CHECK_STAT(ops->read_csr_ne_stat, state->ringnestat, "ringnestat",
 602                          base, bank);
 603         if (ret)
 604                 return ret;
 605
 606         ret = CHECK_STAT(ops->read_csr_nf_stat, state->ringnfstat, "ringnfstat",
 607                          base, bank);
 608         if (ret)
 609                 return ret;
 610
 611         ret = CHECK_STAT(ops->read_csr_f_stat, state->ringfstat, "ringfstat",
 612                          base, bank);
 613         if (ret)
 614                 return ret;
 615
 616         ret = CHECK_STAT(ops->read_csr_c_stat, state->ringcstat0, "ringcstat",
 617                          base, bank);
 618         if (ret)
 619                 return ret;
 620
 621         tmp_val = ops->read_csr_exp_stat(base, bank);
 622         val = state->ringexpstat;
 623         if (tmp_val && !val) {
 624                 pr_err("QAT: Bank was restored with exception: 0x%x\n", val);
 625                 return -EINVAL;
 626         }
 627
 628         return 0;
 629 }
 630
 631 int adf_gen4_bank_state_save(struct adf_accel_dev *accel_dev, u32 bank_number,
 632                              struct bank_state *state)
 633 {
 634         struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev);
 635         struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev);
 636         void __iomem *csr_base = adf_get_etr_base(accel_dev);
 637
 638         if (bank_number >= hw_data->num_banks || !state)
 639                 return -EINVAL;
 640
 641         dev_dbg(&GET_DEV(accel_dev), "Saving state of bank %d\n", bank_number);
 642
 643         bank_state_save(csr_ops, csr_base, bank_number, state,
 644                         hw_data->num_rings_per_bank);
 645
 646         return 0;
 647 }
 648 EXPORT_SYMBOL_GPL(adf_gen4_bank_state_save);
 649
 650 int adf_gen4_bank_state_restore(struct adf_accel_dev *accel_dev, u32 bank_number,
 651                                 struct bank_state *state)
 652 {
 653         struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev);
 654         struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev);
 655         void __iomem *csr_base = adf_get_etr_base(accel_dev);
 656         int ret;
 657
 658         if (bank_number >= hw_data->num_banks  || !state)
 659                 return -EINVAL;
 660
 661         dev_dbg(&GET_DEV(accel_dev), "Restoring state of bank %d\n", bank_number);
 662
 663         ret = bank_state_restore(csr_ops, csr_base, bank_number, state,
 664                                  hw_data->num_rings_per_bank, hw_data->tx_rx_gap);
 665         if (ret)
 666                 dev_err(&GET_DEV(accel_dev),
 667                         "Unable to restore state of bank %d\n", bank_number);
 668
 669         return ret;
 670 }
 671 EXPORT_SYMBOL_GPL(adf_gen4_bank_state_restore);