drivers/net/wireless/intel/iwlwifi/iwl-io.c

   1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
   2 /*
   3  * Copyright (C) 2003-2014, 2018-2022 Intel Corporation
   4  * Copyright (C) 2015-2016 Intel Deutschland GmbH
   5  */
   6 #include <linux/delay.h>
   7 #include <linux/device.h>
   8 #include <linux/export.h>
   9
  10 #include "iwl-drv.h"
  11 #include "iwl-io.h"
  12 #include "iwl-csr.h"
  13 #include "iwl-debug.h"
  14 #include "iwl-prph.h"
  15 #include "iwl-fh.h"
  16
  17 void iwl_write8(struct iwl_trans *trans, u32 ofs, u8 val)
  18 {
  19         trace_iwlwifi_dev_iowrite8(trans->dev, ofs, val);
  20         iwl_trans_write8(trans, ofs, val);
  21 }
  22 IWL_EXPORT_SYMBOL(iwl_write8);
  23
  24 void iwl_write32(struct iwl_trans *trans, u32 ofs, u32 val)
  25 {
  26         trace_iwlwifi_dev_iowrite32(trans->dev, ofs, val);
  27         iwl_trans_write32(trans, ofs, val);
  28 }
  29 IWL_EXPORT_SYMBOL(iwl_write32);
  30
  31 void iwl_write64(struct iwl_trans *trans, u64 ofs, u64 val)
  32 {
  33         trace_iwlwifi_dev_iowrite64(trans->dev, ofs, val);
  34         iwl_trans_write32(trans, ofs, lower_32_bits(val));
  35         iwl_trans_write32(trans, ofs + 4, upper_32_bits(val));
  36 }
  37 IWL_EXPORT_SYMBOL(iwl_write64);
  38
  39 u32 iwl_read32(struct iwl_trans *trans, u32 ofs)
  40 {
  41         u32 val = iwl_trans_read32(trans, ofs);
  42
  43         trace_iwlwifi_dev_ioread32(trans->dev, ofs, val);
  44         return val;
  45 }
  46 IWL_EXPORT_SYMBOL(iwl_read32);
  47
  48 #define IWL_POLL_INTERVAL 10    /* microseconds */
  49
  50 int iwl_poll_bit(struct iwl_trans *trans, u32 addr,
  51                  u32 bits, u32 mask, int timeout)
  52 {
  53         int t = 0;
  54
  55         do {
  56                 if ((iwl_read32(trans, addr) & mask) == (bits & mask))
  57                         return t;
  58                 udelay(IWL_POLL_INTERVAL);
  59                 t += IWL_POLL_INTERVAL;
  60         } while (t < timeout);
  61
  62         return -ETIMEDOUT;
  63 }
  64 IWL_EXPORT_SYMBOL(iwl_poll_bit);
  65
  66 u32 iwl_read_direct32(struct iwl_trans *trans, u32 reg)
  67 {
  68         if (iwl_trans_grab_nic_access(trans)) {
  69                 u32 value = iwl_read32(trans, reg);
  70
  71                 iwl_trans_release_nic_access(trans);
  72                 return value;
  73         }
  74
  75         /* return as if we have a HW timeout/failure */
  76         return 0x5a5a5a5a;
  77 }
  78 IWL_EXPORT_SYMBOL(iwl_read_direct32);
  79
  80 void iwl_write_direct32(struct iwl_trans *trans, u32 reg, u32 value)
  81 {
  82         if (iwl_trans_grab_nic_access(trans)) {
  83                 iwl_write32(trans, reg, value);
  84                 iwl_trans_release_nic_access(trans);
  85         }
  86 }
  87 IWL_EXPORT_SYMBOL(iwl_write_direct32);
  88
  89 void iwl_write_direct64(struct iwl_trans *trans, u64 reg, u64 value)
  90 {
  91         if (iwl_trans_grab_nic_access(trans)) {
  92                 iwl_write64(trans, reg, value);
  93                 iwl_trans_release_nic_access(trans);
  94         }
  95 }
  96 IWL_EXPORT_SYMBOL(iwl_write_direct64);
  97
  98 int iwl_poll_direct_bit(struct iwl_trans *trans, u32 addr, u32 mask,
  99                         int timeout)
 100 {
 101         int t = 0;
 102
 103         do {
 104                 if ((iwl_read_direct32(trans, addr) & mask) == mask)
 105                         return t;
 106                 udelay(IWL_POLL_INTERVAL);
 107                 t += IWL_POLL_INTERVAL;
 108         } while (t < timeout);
 109
 110         return -ETIMEDOUT;
 111 }
 112 IWL_EXPORT_SYMBOL(iwl_poll_direct_bit);
 113
 114 u32 iwl_read_prph_no_grab(struct iwl_trans *trans, u32 ofs)
 115 {
 116         u32 val = iwl_trans_read_prph(trans, ofs);
 117         trace_iwlwifi_dev_ioread_prph32(trans->dev, ofs, val);
 118         return val;
 119 }
 120 IWL_EXPORT_SYMBOL(iwl_read_prph_no_grab);
 121
 122 void iwl_write_prph_no_grab(struct iwl_trans *trans, u32 ofs, u32 val)
 123 {
 124         trace_iwlwifi_dev_iowrite_prph32(trans->dev, ofs, val);
 125         iwl_trans_write_prph(trans, ofs, val);
 126 }
 127 IWL_EXPORT_SYMBOL(iwl_write_prph_no_grab);
 128
 129 void iwl_write_prph64_no_grab(struct iwl_trans *trans, u64 ofs, u64 val)
 130 {
 131         trace_iwlwifi_dev_iowrite_prph64(trans->dev, ofs, val);
 132         iwl_write_prph_no_grab(trans, ofs, val & 0xffffffff);
 133         iwl_write_prph_no_grab(trans, ofs + 4, val >> 32);
 134 }
 135 IWL_EXPORT_SYMBOL(iwl_write_prph64_no_grab);
 136
 137 u32 iwl_read_prph(struct iwl_trans *trans, u32 ofs)
 138 {
 139         if (iwl_trans_grab_nic_access(trans)) {
 140                 u32 val = iwl_read_prph_no_grab(trans, ofs);
 141
 142                 iwl_trans_release_nic_access(trans);
 143
 144                 return val;
 145         }
 146
 147         /* return as if we have a HW timeout/failure */
 148         return 0x5a5a5a5a;
 149 }
 150 IWL_EXPORT_SYMBOL(iwl_read_prph);
 151
 152 void iwl_write_prph_delay(struct iwl_trans *trans, u32 ofs, u32 val, u32 delay_ms)
 153 {
 154         if (iwl_trans_grab_nic_access(trans)) {
 155                 mdelay(delay_ms);
 156                 iwl_write_prph_no_grab(trans, ofs, val);
 157                 iwl_trans_release_nic_access(trans);
 158         }
 159 }
 160 IWL_EXPORT_SYMBOL(iwl_write_prph_delay);
 161
 162 int iwl_poll_prph_bit(struct iwl_trans *trans, u32 addr,
 163                       u32 bits, u32 mask, int timeout)
 164 {
 165         int t = 0;
 166
 167         do {
 168                 if ((iwl_read_prph(trans, addr) & mask) == (bits & mask))
 169                         return t;
 170                 udelay(IWL_POLL_INTERVAL);
 171                 t += IWL_POLL_INTERVAL;
 172         } while (t < timeout);
 173
 174         return -ETIMEDOUT;
 175 }
 176
 177 void iwl_set_bits_prph(struct iwl_trans *trans, u32 ofs, u32 mask)
 178 {
 179         if (iwl_trans_grab_nic_access(trans)) {
 180                 iwl_write_prph_no_grab(trans, ofs,
 181                                        iwl_read_prph_no_grab(trans, ofs) |
 182                                        mask);
 183                 iwl_trans_release_nic_access(trans);
 184         }
 185 }
 186 IWL_EXPORT_SYMBOL(iwl_set_bits_prph);
 187
 188 void iwl_set_bits_mask_prph(struct iwl_trans *trans, u32 ofs,
 189                             u32 bits, u32 mask)
 190 {
 191         if (iwl_trans_grab_nic_access(trans)) {
 192                 iwl_write_prph_no_grab(trans, ofs,
 193                                        (iwl_read_prph_no_grab(trans, ofs) &
 194                                         mask) | bits);
 195                 iwl_trans_release_nic_access(trans);
 196         }
 197 }
 198 IWL_EXPORT_SYMBOL(iwl_set_bits_mask_prph);
 199
 200 void iwl_clear_bits_prph(struct iwl_trans *trans, u32 ofs, u32 mask)
 201 {
 202         u32 val;
 203
 204         if (iwl_trans_grab_nic_access(trans)) {
 205                 val = iwl_read_prph_no_grab(trans, ofs);
 206                 iwl_write_prph_no_grab(trans, ofs, (val & ~mask));
 207                 iwl_trans_release_nic_access(trans);
 208         }
 209 }
 210 IWL_EXPORT_SYMBOL(iwl_clear_bits_prph);
 211
 212 void iwl_force_nmi(struct iwl_trans *trans)
 213 {
 214         if (trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_9000)
 215                 iwl_write_prph_delay(trans, DEVICE_SET_NMI_REG,
 216                                      DEVICE_SET_NMI_VAL_DRV, 1);
 217         else if (trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_AX210)
 218                 iwl_write_umac_prph(trans, UREG_NIC_SET_NMI_DRIVER,
 219                                 UREG_NIC_SET_NMI_DRIVER_NMI_FROM_DRIVER);
 220         else if (trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_BZ)
 221                 iwl_write_umac_prph(trans, UREG_DOORBELL_TO_ISR6,
 222                                     UREG_DOORBELL_TO_ISR6_NMI_BIT);
 223         else
 224                 iwl_write32(trans, CSR_DOORBELL_VECTOR,
 225                             UREG_DOORBELL_TO_ISR6_NMI_BIT);
 226 }
 227 IWL_EXPORT_SYMBOL(iwl_force_nmi);
 228
 229 static const char *get_rfh_string(int cmd)
 230 {
 231 #define IWL_CMD(x) case x: return #x
 232 #define IWL_CMD_MQ(arg, reg, q) { if (arg == reg(q)) return #reg; }
 233
 234         int i;
 235
 236         for (i = 0; i < IWL_MAX_RX_HW_QUEUES; i++) {
 237                 IWL_CMD_MQ(cmd, RFH_Q_FRBDCB_BA_LSB, i);
 238                 IWL_CMD_MQ(cmd, RFH_Q_FRBDCB_WIDX, i);
 239                 IWL_CMD_MQ(cmd, RFH_Q_FRBDCB_RIDX, i);
 240                 IWL_CMD_MQ(cmd, RFH_Q_URBD_STTS_WPTR_LSB, i);
 241         }
 242
 243         switch (cmd) {
 244         IWL_CMD(RFH_RXF_DMA_CFG);
 245         IWL_CMD(RFH_GEN_CFG);
 246         IWL_CMD(RFH_GEN_STATUS);
 247         IWL_CMD(FH_TSSR_TX_STATUS_REG);
 248         IWL_CMD(FH_TSSR_TX_ERROR_REG);
 249         default:
 250                 return "UNKNOWN";
 251         }
 252 #undef IWL_CMD_MQ
 253 }
 254
 255 struct reg {
 256         u32 addr;
 257         bool is64;
 258 };
 259
 260 static int iwl_dump_rfh(struct iwl_trans *trans, char **buf)
 261 {
 262         int i, q;
 263         int num_q = trans->num_rx_queues;
 264         static const u32 rfh_tbl[] = {
 265                 RFH_RXF_DMA_CFG,
 266                 RFH_GEN_CFG,
 267                 RFH_GEN_STATUS,
 268                 FH_TSSR_TX_STATUS_REG,
 269                 FH_TSSR_TX_ERROR_REG,
 270         };
 271         static const struct reg rfh_mq_tbl[] = {
 272                 { RFH_Q0_FRBDCB_BA_LSB, true },
 273                 { RFH_Q0_FRBDCB_WIDX, false },
 274                 { RFH_Q0_FRBDCB_RIDX, false },
 275                 { RFH_Q0_URBD_STTS_WPTR_LSB, true },
 276         };
 277
 278 #ifdef CONFIG_IWLWIFI_DEBUGFS
 279         if (buf) {
 280                 int pos = 0;
 281                 /*
 282                  * Register (up to 34 for name + 8 blank/q for MQ): 40 chars
 283                  * Colon + space: 2 characters
 284                  * 0X%08x: 10 characters
 285                  * New line: 1 character
 286                  * Total of 53 characters
 287                  */
 288                 size_t bufsz = ARRAY_SIZE(rfh_tbl) * 53 +
 289                                ARRAY_SIZE(rfh_mq_tbl) * 53 * num_q + 40;
 290
 291                 *buf = kmalloc(bufsz, GFP_KERNEL);
 292                 if (!*buf)
 293                         return -ENOMEM;
 294
 295                 pos += scnprintf(*buf + pos, bufsz - pos,
 296                                 "RFH register values:\n");
 297
 298                 for (i = 0; i < ARRAY_SIZE(rfh_tbl); i++)
 299                         pos += scnprintf(*buf + pos, bufsz - pos,
 300                                 "%40s: 0X%08x\n",
 301                                 get_rfh_string(rfh_tbl[i]),
 302                                 iwl_read_prph(trans, rfh_tbl[i]));
 303
 304                 for (i = 0; i < ARRAY_SIZE(rfh_mq_tbl); i++)
 305                         for (q = 0; q < num_q; q++) {
 306                                 u32 addr = rfh_mq_tbl[i].addr;
 307
 308                                 addr += q * (rfh_mq_tbl[i].is64 ? 8 : 4);
 309                                 pos += scnprintf(*buf + pos, bufsz - pos,
 310                                         "%34s(q %2d): 0X%08x\n",
 311                                         get_rfh_string(addr), q,
 312                                         iwl_read_prph(trans, addr));
 313                         }
 314
 315                 return pos;
 316         }
 317 #endif
 318
 319         IWL_ERR(trans, "RFH register values:\n");
 320         for (i = 0; i < ARRAY_SIZE(rfh_tbl); i++)
 321                 IWL_ERR(trans, "  %34s: 0X%08x\n",
 322                         get_rfh_string(rfh_tbl[i]),
 323                         iwl_read_prph(trans, rfh_tbl[i]));
 324
 325         for (i = 0; i < ARRAY_SIZE(rfh_mq_tbl); i++)
 326                 for (q = 0; q < num_q; q++) {
 327                         u32 addr = rfh_mq_tbl[i].addr;
 328
 329                         addr += q * (rfh_mq_tbl[i].is64 ? 8 : 4);
 330                         IWL_ERR(trans, "  %34s(q %d): 0X%08x\n",
 331                                 get_rfh_string(addr), q,
 332                                 iwl_read_prph(trans, addr));
 333                 }
 334
 335         return 0;
 336 }
 337
 338 static const char *get_fh_string(int cmd)
 339 {
 340         switch (cmd) {
 341         IWL_CMD(FH_RSCSR_CHNL0_STTS_WPTR_REG);
 342         IWL_CMD(FH_RSCSR_CHNL0_RBDCB_BASE_REG);
 343         IWL_CMD(FH_RSCSR_CHNL0_WPTR);
 344         IWL_CMD(FH_MEM_RCSR_CHNL0_CONFIG_REG);
 345         IWL_CMD(FH_MEM_RSSR_SHARED_CTRL_REG);
 346         IWL_CMD(FH_MEM_RSSR_RX_STATUS_REG);
 347         IWL_CMD(FH_MEM_RSSR_RX_ENABLE_ERR_IRQ2DRV);
 348         IWL_CMD(FH_TSSR_TX_STATUS_REG);
 349         IWL_CMD(FH_TSSR_TX_ERROR_REG);
 350         default:
 351                 return "UNKNOWN";
 352         }
 353 #undef IWL_CMD
 354 }
 355
 356 int iwl_dump_fh(struct iwl_trans *trans, char **buf)
 357 {
 358         int i;
 359         static const u32 fh_tbl[] = {
 360                 FH_RSCSR_CHNL0_STTS_WPTR_REG,
 361                 FH_RSCSR_CHNL0_RBDCB_BASE_REG,
 362                 FH_RSCSR_CHNL0_WPTR,
 363                 FH_MEM_RCSR_CHNL0_CONFIG_REG,
 364                 FH_MEM_RSSR_SHARED_CTRL_REG,
 365                 FH_MEM_RSSR_RX_STATUS_REG,
 366                 FH_MEM_RSSR_RX_ENABLE_ERR_IRQ2DRV,
 367                 FH_TSSR_TX_STATUS_REG,
 368                 FH_TSSR_TX_ERROR_REG
 369         };
 370
 371         if (trans->trans_cfg->mq_rx_supported)
 372                 return iwl_dump_rfh(trans, buf);
 373
 374 #ifdef CONFIG_IWLWIFI_DEBUGFS
 375         if (buf) {
 376                 int pos = 0;
 377                 size_t bufsz = ARRAY_SIZE(fh_tbl) * 48 + 40;
 378
 379                 *buf = kmalloc(bufsz, GFP_KERNEL);
 380                 if (!*buf)
 381                         return -ENOMEM;
 382
 383                 pos += scnprintf(*buf + pos, bufsz - pos,
 384                                 "FH register values:\n");
 385
 386                 for (i = 0; i < ARRAY_SIZE(fh_tbl); i++)
 387                         pos += scnprintf(*buf + pos, bufsz - pos,
 388                                 "  %34s: 0X%08x\n",
 389                                 get_fh_string(fh_tbl[i]),
 390                                 iwl_read_direct32(trans, fh_tbl[i]));
 391
 392                 return pos;
 393         }
 394 #endif
 395
 396         IWL_ERR(trans, "FH register values:\n");
 397         for (i = 0; i <  ARRAY_SIZE(fh_tbl); i++)
 398                 IWL_ERR(trans, "  %34s: 0X%08x\n",
 399                         get_fh_string(fh_tbl[i]),
 400                         iwl_read_direct32(trans, fh_tbl[i]));
 401
 402         return 0;
 403 }
 404
 405 #define IWL_HOST_MON_BLOCK_PEMON        0x00
 406 #define IWL_HOST_MON_BLOCK_HIPM         0x22
 407
 408 #define IWL_HOST_MON_BLOCK_PEMON_VEC0   0x00
 409 #define IWL_HOST_MON_BLOCK_PEMON_VEC1   0x01
 410 #define IWL_HOST_MON_BLOCK_PEMON_WFPM   0x06
 411
 412 static void iwl_dump_host_monitor_block(struct iwl_trans *trans,
 413                                         u32 block, u32 vec, u32 iter)
 414 {
 415         int i;
 416
 417         IWL_ERR(trans, "Host monitor block 0x%x vector 0x%x\n", block, vec);
 418         iwl_write32(trans, CSR_MONITOR_CFG_REG, (block << 8) | vec);
 419         for (i = 0; i < iter; i++)
 420                 IWL_ERR(trans, "    value [iter %d]: 0x%08x\n",
 421                         i, iwl_read32(trans, CSR_MONITOR_STATUS_REG));
 422 }
 423
 424 static void iwl_dump_host_monitor(struct iwl_trans *trans)
 425 {
 426         switch (trans->trans_cfg->device_family) {
 427         case IWL_DEVICE_FAMILY_22000:
 428         case IWL_DEVICE_FAMILY_AX210:
 429                 IWL_ERR(trans, "CSR_RESET = 0x%x\n",
 430                         iwl_read32(trans, CSR_RESET));
 431                 iwl_dump_host_monitor_block(trans, IWL_HOST_MON_BLOCK_PEMON,
 432                                             IWL_HOST_MON_BLOCK_PEMON_VEC0, 15);
 433                 iwl_dump_host_monitor_block(trans, IWL_HOST_MON_BLOCK_PEMON,
 434                                             IWL_HOST_MON_BLOCK_PEMON_VEC1, 15);
 435                 iwl_dump_host_monitor_block(trans, IWL_HOST_MON_BLOCK_PEMON,
 436                                             IWL_HOST_MON_BLOCK_PEMON_WFPM, 15);
 437                 iwl_dump_host_monitor_block(trans, IWL_HOST_MON_BLOCK_HIPM,
 438                                             IWL_HOST_MON_BLOCK_PEMON_VEC0, 1);
 439                 break;
 440         default:
 441                 /* not supported yet */
 442                 return;
 443         }
 444 }
 445
 446 int iwl_finish_nic_init(struct iwl_trans *trans)
 447 {
 448         const struct iwl_cfg_trans_params *cfg_trans = trans->trans_cfg;
 449         u32 poll_ready;
 450         int err;
 451
 452         if (cfg_trans->bisr_workaround) {
 453                 /* ensure the TOP FSM isn't still in previous reset */
 454                 mdelay(2);
 455         }
 456
 457         /*
 458          * Set "initialization complete" bit to move adapter from
 459          * D0U* --> D0A* (powered-up active) state.
 460          */
 461         if (cfg_trans->device_family >= IWL_DEVICE_FAMILY_BZ) {
 462                 iwl_set_bit(trans, CSR_GP_CNTRL,
 463                             CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY |
 464                             CSR_GP_CNTRL_REG_FLAG_MAC_INIT);
 465                 poll_ready = CSR_GP_CNTRL_REG_FLAG_MAC_STATUS;
 466         } else {
 467                 iwl_set_bit(trans, CSR_GP_CNTRL,
 468                             CSR_GP_CNTRL_REG_FLAG_INIT_DONE);
 469                 poll_ready = CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY;
 470         }
 471
 472         if (cfg_trans->device_family == IWL_DEVICE_FAMILY_8000)
 473                 udelay(2);
 474
 475         /*
 476          * Wait for clock stabilization; once stabilized, access to
 477          * device-internal resources is supported, e.g. iwl_write_prph()
 478          * and accesses to uCode SRAM.
 479          */
 480         err = iwl_poll_bit(trans, CSR_GP_CNTRL, poll_ready, poll_ready, 25000);
 481         if (err < 0) {
 482                 IWL_DEBUG_INFO(trans, "Failed to wake NIC\n");
 483
 484                 iwl_dump_host_monitor(trans);
 485         }
 486
 487         if (cfg_trans->bisr_workaround) {
 488                 /* ensure BISR shift has finished */
 489                 udelay(200);
 490         }
 491
 492         return err < 0 ? err : 0;
 493 }
 494 IWL_EXPORT_SYMBOL(iwl_finish_nic_init);
 495
 496 void iwl_trans_sync_nmi_with_addr(struct iwl_trans *trans, u32 inta_addr,
 497                                   u32 sw_err_bit)
 498 {
 499         unsigned long timeout = jiffies + IWL_TRANS_NMI_TIMEOUT;
 500         bool interrupts_enabled = test_bit(STATUS_INT_ENABLED, &trans->status);
 501
 502         /* if the interrupts were already disabled, there is no point in
 503          * calling iwl_disable_interrupts
 504          */
 505         if (interrupts_enabled)
 506                 iwl_trans_interrupts(trans, false);
 507
 508         iwl_force_nmi(trans);
 509         while (time_after(timeout, jiffies)) {
 510                 u32 inta_hw = iwl_read32(trans, inta_addr);
 511
 512                 /* Error detected by uCode */
 513                 if (inta_hw & sw_err_bit) {
 514                         /* Clear causes register */
 515                         iwl_write32(trans, inta_addr, inta_hw & sw_err_bit);
 516                         break;
 517                 }
 518
 519                 mdelay(1);
 520         }
 521
 522         /* enable interrupts only if there were already enabled before this
 523          * function to avoid a case were the driver enable interrupts before
 524          * proper configurations were made
 525          */
 526         if (interrupts_enabled)
 527                 iwl_trans_interrupts(trans, true);
 528
 529         iwl_trans_fw_error(trans, false);
 530 }