drivers/net/wireless/intel/iwlwifi/iwl-io.c

   1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
   2 /*
   3  * Copyright (C) 2003-2014, 2018-2021 Intel Corporation
   4  * Copyright (C) 2015-2016 Intel Deutschland GmbH
   5  */
   6 #include <linux/delay.h>
   7 #include <linux/device.h>
   8 #include <linux/export.h>
   9
  10 #include "iwl-drv.h"
  11 #include "iwl-io.h"
  12 #include "iwl-csr.h"
  13 #include "iwl-debug.h"
  14 #include "iwl-prph.h"
  15 #include "iwl-fh.h"
  16
  17 void iwl_write8(struct iwl_trans *trans, u32 ofs, u8 val)
  18 {
  19         trace_iwlwifi_dev_iowrite8(trans->dev, ofs, val);
  20         iwl_trans_write8(trans, ofs, val);
  21 }
  22 IWL_EXPORT_SYMBOL(iwl_write8);
  23
  24 void iwl_write32(struct iwl_trans *trans, u32 ofs, u32 val)
  25 {
  26         trace_iwlwifi_dev_iowrite32(trans->dev, ofs, val);
  27         iwl_trans_write32(trans, ofs, val);
  28 }
  29 IWL_EXPORT_SYMBOL(iwl_write32);
  30
  31 void iwl_write64(struct iwl_trans *trans, u64 ofs, u64 val)
  32 {
  33         trace_iwlwifi_dev_iowrite64(trans->dev, ofs, val);
  34         iwl_trans_write32(trans, ofs, lower_32_bits(val));
  35         iwl_trans_write32(trans, ofs + 4, upper_32_bits(val));
  36 }
  37 IWL_EXPORT_SYMBOL(iwl_write64);
  38
  39 u32 iwl_read32(struct iwl_trans *trans, u32 ofs)
  40 {
  41         u32 val = iwl_trans_read32(trans, ofs);
  42
  43         trace_iwlwifi_dev_ioread32(trans->dev, ofs, val);
  44         return val;
  45 }
  46 IWL_EXPORT_SYMBOL(iwl_read32);
  47
  48 #define IWL_POLL_INTERVAL 10    /* microseconds */
  49
  50 int iwl_poll_bit(struct iwl_trans *trans, u32 addr,
  51                  u32 bits, u32 mask, int timeout)
  52 {
  53         int t = 0;
  54
  55         do {
  56                 if ((iwl_read32(trans, addr) & mask) == (bits & mask))
  57                         return t;
  58                 udelay(IWL_POLL_INTERVAL);
  59                 t += IWL_POLL_INTERVAL;
  60         } while (t < timeout);
  61
  62         return -ETIMEDOUT;
  63 }
  64 IWL_EXPORT_SYMBOL(iwl_poll_bit);
  65
  66 u32 iwl_read_direct32(struct iwl_trans *trans, u32 reg)
  67 {
  68         u32 value = 0x5a5a5a5a;
  69
  70         if (iwl_trans_grab_nic_access(trans)) {
  71                 value = iwl_read32(trans, reg);
  72                 iwl_trans_release_nic_access(trans);
  73         }
  74
  75         return value;
  76 }
  77 IWL_EXPORT_SYMBOL(iwl_read_direct32);
  78
  79 void iwl_write_direct32(struct iwl_trans *trans, u32 reg, u32 value)
  80 {
  81         if (iwl_trans_grab_nic_access(trans)) {
  82                 iwl_write32(trans, reg, value);
  83                 iwl_trans_release_nic_access(trans);
  84         }
  85 }
  86 IWL_EXPORT_SYMBOL(iwl_write_direct32);
  87
  88 void iwl_write_direct64(struct iwl_trans *trans, u64 reg, u64 value)
  89 {
  90         if (iwl_trans_grab_nic_access(trans)) {
  91                 iwl_write64(trans, reg, value);
  92                 iwl_trans_release_nic_access(trans);
  93         }
  94 }
  95 IWL_EXPORT_SYMBOL(iwl_write_direct64);
  96
  97 int iwl_poll_direct_bit(struct iwl_trans *trans, u32 addr, u32 mask,
  98                         int timeout)
  99 {
 100         int t = 0;
 101
 102         do {
 103                 if ((iwl_read_direct32(trans, addr) & mask) == mask)
 104                         return t;
 105                 udelay(IWL_POLL_INTERVAL);
 106                 t += IWL_POLL_INTERVAL;
 107         } while (t < timeout);
 108
 109         return -ETIMEDOUT;
 110 }
 111 IWL_EXPORT_SYMBOL(iwl_poll_direct_bit);
 112
 113 u32 iwl_read_prph_no_grab(struct iwl_trans *trans, u32 ofs)
 114 {
 115         u32 val = iwl_trans_read_prph(trans, ofs);
 116         trace_iwlwifi_dev_ioread_prph32(trans->dev, ofs, val);
 117         return val;
 118 }
 119 IWL_EXPORT_SYMBOL(iwl_read_prph_no_grab);
 120
 121 void iwl_write_prph_no_grab(struct iwl_trans *trans, u32 ofs, u32 val)
 122 {
 123         trace_iwlwifi_dev_iowrite_prph32(trans->dev, ofs, val);
 124         iwl_trans_write_prph(trans, ofs, val);
 125 }
 126 IWL_EXPORT_SYMBOL(iwl_write_prph_no_grab);
 127
 128 void iwl_write_prph64_no_grab(struct iwl_trans *trans, u64 ofs, u64 val)
 129 {
 130         trace_iwlwifi_dev_iowrite_prph64(trans->dev, ofs, val);
 131         iwl_write_prph_no_grab(trans, ofs, val & 0xffffffff);
 132         iwl_write_prph_no_grab(trans, ofs + 4, val >> 32);
 133 }
 134 IWL_EXPORT_SYMBOL(iwl_write_prph64_no_grab);
 135
 136 u32 iwl_read_prph(struct iwl_trans *trans, u32 ofs)
 137 {
 138         u32 val = 0x5a5a5a5a;
 139
 140         if (iwl_trans_grab_nic_access(trans)) {
 141                 val = iwl_read_prph_no_grab(trans, ofs);
 142                 iwl_trans_release_nic_access(trans);
 143         }
 144         return val;
 145 }
 146 IWL_EXPORT_SYMBOL(iwl_read_prph);
 147
 148 void iwl_write_prph_delay(struct iwl_trans *trans, u32 ofs, u32 val, u32 delay_ms)
 149 {
 150         if (iwl_trans_grab_nic_access(trans)) {
 151                 mdelay(delay_ms);
 152                 iwl_write_prph_no_grab(trans, ofs, val);
 153                 iwl_trans_release_nic_access(trans);
 154         }
 155 }
 156 IWL_EXPORT_SYMBOL(iwl_write_prph_delay);
 157
 158 int iwl_poll_prph_bit(struct iwl_trans *trans, u32 addr,
 159                       u32 bits, u32 mask, int timeout)
 160 {
 161         int t = 0;
 162
 163         do {
 164                 if ((iwl_read_prph(trans, addr) & mask) == (bits & mask))
 165                         return t;
 166                 udelay(IWL_POLL_INTERVAL);
 167                 t += IWL_POLL_INTERVAL;
 168         } while (t < timeout);
 169
 170         return -ETIMEDOUT;
 171 }
 172
 173 void iwl_set_bits_prph(struct iwl_trans *trans, u32 ofs, u32 mask)
 174 {
 175         if (iwl_trans_grab_nic_access(trans)) {
 176                 iwl_write_prph_no_grab(trans, ofs,
 177                                        iwl_read_prph_no_grab(trans, ofs) |
 178                                        mask);
 179                 iwl_trans_release_nic_access(trans);
 180         }
 181 }
 182 IWL_EXPORT_SYMBOL(iwl_set_bits_prph);
 183
 184 void iwl_set_bits_mask_prph(struct iwl_trans *trans, u32 ofs,
 185                             u32 bits, u32 mask)
 186 {
 187         if (iwl_trans_grab_nic_access(trans)) {
 188                 iwl_write_prph_no_grab(trans, ofs,
 189                                        (iwl_read_prph_no_grab(trans, ofs) &
 190                                         mask) | bits);
 191                 iwl_trans_release_nic_access(trans);
 192         }
 193 }
 194 IWL_EXPORT_SYMBOL(iwl_set_bits_mask_prph);
 195
 196 void iwl_clear_bits_prph(struct iwl_trans *trans, u32 ofs, u32 mask)
 197 {
 198         u32 val;
 199
 200         if (iwl_trans_grab_nic_access(trans)) {
 201                 val = iwl_read_prph_no_grab(trans, ofs);
 202                 iwl_write_prph_no_grab(trans, ofs, (val & ~mask));
 203                 iwl_trans_release_nic_access(trans);
 204         }
 205 }
 206 IWL_EXPORT_SYMBOL(iwl_clear_bits_prph);
 207
 208 void iwl_force_nmi(struct iwl_trans *trans)
 209 {
 210         if (trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_9000)
 211                 iwl_write_prph_delay(trans, DEVICE_SET_NMI_REG,
 212                                      DEVICE_SET_NMI_VAL_DRV, 1);
 213         else if (trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_AX210)
 214                 iwl_write_umac_prph(trans, UREG_NIC_SET_NMI_DRIVER,
 215                                 UREG_NIC_SET_NMI_DRIVER_NMI_FROM_DRIVER);
 216         else if (trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_BZ)
 217                 iwl_write_umac_prph(trans, UREG_DOORBELL_TO_ISR6,
 218                                     UREG_DOORBELL_TO_ISR6_NMI_BIT);
 219         else
 220                 iwl_write32(trans, CSR_DOORBELL_VECTOR,
 221                             CSR_DOORBELL_VECTOR_NMI);
 222 }
 223 IWL_EXPORT_SYMBOL(iwl_force_nmi);
 224
 225 static const char *get_rfh_string(int cmd)
 226 {
 227 #define IWL_CMD(x) case x: return #x
 228 #define IWL_CMD_MQ(arg, reg, q) { if (arg == reg(q)) return #reg; }
 229
 230         int i;
 231
 232         for (i = 0; i < IWL_MAX_RX_HW_QUEUES; i++) {
 233                 IWL_CMD_MQ(cmd, RFH_Q_FRBDCB_BA_LSB, i);
 234                 IWL_CMD_MQ(cmd, RFH_Q_FRBDCB_WIDX, i);
 235                 IWL_CMD_MQ(cmd, RFH_Q_FRBDCB_RIDX, i);
 236                 IWL_CMD_MQ(cmd, RFH_Q_URBD_STTS_WPTR_LSB, i);
 237         }
 238
 239         switch (cmd) {
 240         IWL_CMD(RFH_RXF_DMA_CFG);
 241         IWL_CMD(RFH_GEN_CFG);
 242         IWL_CMD(RFH_GEN_STATUS);
 243         IWL_CMD(FH_TSSR_TX_STATUS_REG);
 244         IWL_CMD(FH_TSSR_TX_ERROR_REG);
 245         default:
 246                 return "UNKNOWN";
 247         }
 248 #undef IWL_CMD_MQ
 249 }
 250
 251 struct reg {
 252         u32 addr;
 253         bool is64;
 254 };
 255
 256 static int iwl_dump_rfh(struct iwl_trans *trans, char **buf)
 257 {
 258         int i, q;
 259         int num_q = trans->num_rx_queues;
 260         static const u32 rfh_tbl[] = {
 261                 RFH_RXF_DMA_CFG,
 262                 RFH_GEN_CFG,
 263                 RFH_GEN_STATUS,
 264                 FH_TSSR_TX_STATUS_REG,
 265                 FH_TSSR_TX_ERROR_REG,
 266         };
 267         static const struct reg rfh_mq_tbl[] = {
 268                 { RFH_Q0_FRBDCB_BA_LSB, true },
 269                 { RFH_Q0_FRBDCB_WIDX, false },
 270                 { RFH_Q0_FRBDCB_RIDX, false },
 271                 { RFH_Q0_URBD_STTS_WPTR_LSB, true },
 272         };
 273
 274 #ifdef CONFIG_IWLWIFI_DEBUGFS
 275         if (buf) {
 276                 int pos = 0;
 277                 /*
 278                  * Register (up to 34 for name + 8 blank/q for MQ): 40 chars
 279                  * Colon + space: 2 characters
 280                  * 0X%08x: 10 characters
 281                  * New line: 1 character
 282                  * Total of 53 characters
 283                  */
 284                 size_t bufsz = ARRAY_SIZE(rfh_tbl) * 53 +
 285                                ARRAY_SIZE(rfh_mq_tbl) * 53 * num_q + 40;
 286
 287                 *buf = kmalloc(bufsz, GFP_KERNEL);
 288                 if (!*buf)
 289                         return -ENOMEM;
 290
 291                 pos += scnprintf(*buf + pos, bufsz - pos,
 292                                 "RFH register values:\n");
 293
 294                 for (i = 0; i < ARRAY_SIZE(rfh_tbl); i++)
 295                         pos += scnprintf(*buf + pos, bufsz - pos,
 296                                 "%40s: 0X%08x\n",
 297                                 get_rfh_string(rfh_tbl[i]),
 298                                 iwl_read_prph(trans, rfh_tbl[i]));
 299
 300                 for (i = 0; i < ARRAY_SIZE(rfh_mq_tbl); i++)
 301                         for (q = 0; q < num_q; q++) {
 302                                 u32 addr = rfh_mq_tbl[i].addr;
 303
 304                                 addr += q * (rfh_mq_tbl[i].is64 ? 8 : 4);
 305                                 pos += scnprintf(*buf + pos, bufsz - pos,
 306                                         "%34s(q %2d): 0X%08x\n",
 307                                         get_rfh_string(addr), q,
 308                                         iwl_read_prph(trans, addr));
 309                         }
 310
 311                 return pos;
 312         }
 313 #endif
 314
 315         IWL_ERR(trans, "RFH register values:\n");
 316         for (i = 0; i < ARRAY_SIZE(rfh_tbl); i++)
 317                 IWL_ERR(trans, "  %34s: 0X%08x\n",
 318                         get_rfh_string(rfh_tbl[i]),
 319                         iwl_read_prph(trans, rfh_tbl[i]));
 320
 321         for (i = 0; i < ARRAY_SIZE(rfh_mq_tbl); i++)
 322                 for (q = 0; q < num_q; q++) {
 323                         u32 addr = rfh_mq_tbl[i].addr;
 324
 325                         addr += q * (rfh_mq_tbl[i].is64 ? 8 : 4);
 326                         IWL_ERR(trans, "  %34s(q %d): 0X%08x\n",
 327                                 get_rfh_string(addr), q,
 328                                 iwl_read_prph(trans, addr));
 329                 }
 330
 331         return 0;
 332 }
 333
 334 static const char *get_fh_string(int cmd)
 335 {
 336         switch (cmd) {
 337         IWL_CMD(FH_RSCSR_CHNL0_STTS_WPTR_REG);
 338         IWL_CMD(FH_RSCSR_CHNL0_RBDCB_BASE_REG);
 339         IWL_CMD(FH_RSCSR_CHNL0_WPTR);
 340         IWL_CMD(FH_MEM_RCSR_CHNL0_CONFIG_REG);
 341         IWL_CMD(FH_MEM_RSSR_SHARED_CTRL_REG);
 342         IWL_CMD(FH_MEM_RSSR_RX_STATUS_REG);
 343         IWL_CMD(FH_MEM_RSSR_RX_ENABLE_ERR_IRQ2DRV);
 344         IWL_CMD(FH_TSSR_TX_STATUS_REG);
 345         IWL_CMD(FH_TSSR_TX_ERROR_REG);
 346         default:
 347                 return "UNKNOWN";
 348         }
 349 #undef IWL_CMD
 350 }
 351
 352 int iwl_dump_fh(struct iwl_trans *trans, char **buf)
 353 {
 354         int i;
 355         static const u32 fh_tbl[] = {
 356                 FH_RSCSR_CHNL0_STTS_WPTR_REG,
 357                 FH_RSCSR_CHNL0_RBDCB_BASE_REG,
 358                 FH_RSCSR_CHNL0_WPTR,
 359                 FH_MEM_RCSR_CHNL0_CONFIG_REG,
 360                 FH_MEM_RSSR_SHARED_CTRL_REG,
 361                 FH_MEM_RSSR_RX_STATUS_REG,
 362                 FH_MEM_RSSR_RX_ENABLE_ERR_IRQ2DRV,
 363                 FH_TSSR_TX_STATUS_REG,
 364                 FH_TSSR_TX_ERROR_REG
 365         };
 366
 367         if (trans->trans_cfg->mq_rx_supported)
 368                 return iwl_dump_rfh(trans, buf);
 369
 370 #ifdef CONFIG_IWLWIFI_DEBUGFS
 371         if (buf) {
 372                 int pos = 0;
 373                 size_t bufsz = ARRAY_SIZE(fh_tbl) * 48 + 40;
 374
 375                 *buf = kmalloc(bufsz, GFP_KERNEL);
 376                 if (!*buf)
 377                         return -ENOMEM;
 378
 379                 pos += scnprintf(*buf + pos, bufsz - pos,
 380                                 "FH register values:\n");
 381
 382                 for (i = 0; i < ARRAY_SIZE(fh_tbl); i++)
 383                         pos += scnprintf(*buf + pos, bufsz - pos,
 384                                 "  %34s: 0X%08x\n",
 385                                 get_fh_string(fh_tbl[i]),
 386                                 iwl_read_direct32(trans, fh_tbl[i]));
 387
 388                 return pos;
 389         }
 390 #endif
 391
 392         IWL_ERR(trans, "FH register values:\n");
 393         for (i = 0; i <  ARRAY_SIZE(fh_tbl); i++)
 394                 IWL_ERR(trans, "  %34s: 0X%08x\n",
 395                         get_fh_string(fh_tbl[i]),
 396                         iwl_read_direct32(trans, fh_tbl[i]));
 397
 398         return 0;
 399 }
 400
 401 #define IWL_HOST_MON_BLOCK_PEMON        0x00
 402 #define IWL_HOST_MON_BLOCK_HIPM         0x22
 403
 404 #define IWL_HOST_MON_BLOCK_PEMON_VEC0   0x00
 405 #define IWL_HOST_MON_BLOCK_PEMON_VEC1   0x01
 406 #define IWL_HOST_MON_BLOCK_PEMON_WFPM   0x06
 407
 408 static void iwl_dump_host_monitor_block(struct iwl_trans *trans,
 409                                         u32 block, u32 vec, u32 iter)
 410 {
 411         int i;
 412
 413         IWL_ERR(trans, "Host monitor block 0x%x vector 0x%x\n", block, vec);
 414         iwl_write32(trans, CSR_MONITOR_CFG_REG, (block << 8) | vec);
 415         for (i = 0; i < iter; i++)
 416                 IWL_ERR(trans, "    value [iter %d]: 0x%08x\n",
 417                         i, iwl_read32(trans, CSR_MONITOR_STATUS_REG));
 418 }
 419
 420 static void iwl_dump_host_monitor(struct iwl_trans *trans)
 421 {
 422         switch (trans->trans_cfg->device_family) {
 423         case IWL_DEVICE_FAMILY_22000:
 424         case IWL_DEVICE_FAMILY_AX210:
 425                 IWL_ERR(trans, "CSR_RESET = 0x%x\n",
 426                         iwl_read32(trans, CSR_RESET));
 427                 iwl_dump_host_monitor_block(trans, IWL_HOST_MON_BLOCK_PEMON,
 428                                             IWL_HOST_MON_BLOCK_PEMON_VEC0, 15);
 429                 iwl_dump_host_monitor_block(trans, IWL_HOST_MON_BLOCK_PEMON,
 430                                             IWL_HOST_MON_BLOCK_PEMON_VEC1, 15);
 431                 iwl_dump_host_monitor_block(trans, IWL_HOST_MON_BLOCK_PEMON,
 432                                             IWL_HOST_MON_BLOCK_PEMON_WFPM, 15);
 433                 iwl_dump_host_monitor_block(trans, IWL_HOST_MON_BLOCK_HIPM,
 434                                             IWL_HOST_MON_BLOCK_PEMON_VEC0, 1);
 435                 break;
 436         default:
 437                 /* not supported yet */
 438                 return;
 439         }
 440 }
 441
 442 int iwl_finish_nic_init(struct iwl_trans *trans)
 443 {
 444         const struct iwl_cfg_trans_params *cfg_trans = trans->trans_cfg;
 445         u32 poll_ready;
 446         int err;
 447
 448         if (cfg_trans->bisr_workaround) {
 449                 /* ensure the TOP FSM isn't still in previous reset */
 450                 mdelay(2);
 451         }
 452
 453         /*
 454          * Set "initialization complete" bit to move adapter from
 455          * D0U* --> D0A* (powered-up active) state.
 456          */
 457         if (cfg_trans->device_family >= IWL_DEVICE_FAMILY_BZ) {
 458                 iwl_set_bit(trans, CSR_GP_CNTRL,
 459                             CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY |
 460                             CSR_GP_CNTRL_REG_FLAG_MAC_INIT);
 461                 poll_ready = CSR_GP_CNTRL_REG_FLAG_MAC_STATUS;
 462         } else {
 463                 iwl_set_bit(trans, CSR_GP_CNTRL,
 464                             CSR_GP_CNTRL_REG_FLAG_INIT_DONE);
 465                 poll_ready = CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY;
 466         }
 467
 468         if (cfg_trans->device_family == IWL_DEVICE_FAMILY_8000)
 469                 udelay(2);
 470
 471         /*
 472          * Wait for clock stabilization; once stabilized, access to
 473          * device-internal resources is supported, e.g. iwl_write_prph()
 474          * and accesses to uCode SRAM.
 475          */
 476         err = iwl_poll_bit(trans, CSR_GP_CNTRL, poll_ready, poll_ready, 25000);
 477         if (err < 0) {
 478                 IWL_DEBUG_INFO(trans, "Failed to wake NIC\n");
 479
 480                 iwl_dump_host_monitor(trans);
 481         }
 482
 483         if (cfg_trans->bisr_workaround) {
 484                 /* ensure BISR shift has finished */
 485                 udelay(200);
 486         }
 487
 488         return err < 0 ? err : 0;
 489 }
 490 IWL_EXPORT_SYMBOL(iwl_finish_nic_init);
 491
 492 void iwl_trans_sync_nmi_with_addr(struct iwl_trans *trans, u32 inta_addr,
 493                                   u32 sw_err_bit)
 494 {
 495         unsigned long timeout = jiffies + IWL_TRANS_NMI_TIMEOUT;
 496         bool interrupts_enabled = test_bit(STATUS_INT_ENABLED, &trans->status);
 497
 498         /* if the interrupts were already disabled, there is no point in
 499          * calling iwl_disable_interrupts
 500          */
 501         if (interrupts_enabled)
 502                 iwl_trans_interrupts(trans, false);
 503
 504         iwl_force_nmi(trans);
 505         while (time_after(timeout, jiffies)) {
 506                 u32 inta_hw = iwl_read32(trans, inta_addr);
 507
 508                 /* Error detected by uCode */
 509                 if (inta_hw & sw_err_bit) {
 510                         /* Clear causes register */
 511                         iwl_write32(trans, inta_addr, inta_hw & sw_err_bit);
 512                         break;
 513                 }
 514
 515                 mdelay(1);
 516         }
 517
 518         /* enable interrupts only if there were already enabled before this
 519          * function to avoid a case were the driver enable interrupts before
 520          * proper configurations were made
 521          */
 522         if (interrupts_enabled)
 523                 iwl_trans_interrupts(trans, true);
 524
 525         iwl_trans_fw_error(trans, false);
 526 }