drivers/ufs/core/ufshcd.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Universal Flash Storage Host controller driver Core
   4  * Copyright (C) 2011-2013 Samsung India Software Operations
   5  * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
   6  *
   7  * Authors:
   8  *      Santosh Yaraganavi <[email protected]>
   9  *      Vinayak Holikatti <[email protected]>
  10  */
  11
  12 #include <linux/async.h>
  13 #include <linux/devfreq.h>
  14 #include <linux/nls.h>
  15 #include <linux/of.h>
  16 #include <linux/bitfield.h>
  17 #include <linux/blk-pm.h>
  18 #include <linux/blkdev.h>
  19 #include <linux/clk.h>
  20 #include <linux/delay.h>
  21 #include <linux/interrupt.h>
  22 #include <linux/module.h>
  23 #include <linux/pm_opp.h>
  24 #include <linux/regulator/consumer.h>
  25 #include <linux/sched/clock.h>
  26 #include <linux/iopoll.h>
  27 #include <scsi/scsi_cmnd.h>
  28 #include <scsi/scsi_dbg.h>
  29 #include <scsi/scsi_driver.h>
  30 #include <scsi/scsi_eh.h>
  31 #include "ufshcd-priv.h"
  32 #include <ufs/ufs_quirks.h>
  33 #include <ufs/unipro.h>
  34 #include "ufs-sysfs.h"
  35 #include "ufs-debugfs.h"
  36 #include "ufs-fault-injection.h"
  37 #include "ufs_bsg.h"
  38 #include "ufshcd-crypto.h"
  39 #include <linux/unaligned.h>
  40
  41 #define CREATE_TRACE_POINTS
  42 #include "ufs_trace.h"
  43
  44 #define UFSHCD_ENABLE_INTRS     (UTP_TRANSFER_REQ_COMPL |\
  45                                  UTP_TASK_REQ_COMPL |\
  46                                  UFSHCD_ERROR_MASK)
  47
  48 #define UFSHCD_ENABLE_MCQ_INTRS (UTP_TASK_REQ_COMPL |\
  49                                  UFSHCD_ERROR_MASK |\
  50                                  MCQ_CQ_EVENT_STATUS)
  51
  52
  53 /* UIC command timeout, unit: ms */
  54 enum {
  55         UIC_CMD_TIMEOUT_DEFAULT = 500,
  56         UIC_CMD_TIMEOUT_MAX     = 2000,
  57 };
  58 /* NOP OUT retries waiting for NOP IN response */
  59 #define NOP_OUT_RETRIES    10
  60 /* Timeout after 50 msecs if NOP OUT hangs without response */
  61 #define NOP_OUT_TIMEOUT    50 /* msecs */
  62
  63 /* Query request retries */
  64 #define QUERY_REQ_RETRIES 3
  65 /* Query request timeout */
  66 #define QUERY_REQ_TIMEOUT 1500 /* 1.5 seconds */
  67
  68 /* Advanced RPMB request timeout */
  69 #define ADVANCED_RPMB_REQ_TIMEOUT  3000 /* 3 seconds */
  70
  71 /* Task management command timeout */
  72 #define TM_CMD_TIMEOUT  100 /* msecs */
  73
  74 /* maximum number of retries for a general UIC command  */
  75 #define UFS_UIC_COMMAND_RETRIES 3
  76
  77 /* maximum number of link-startup retries */
  78 #define DME_LINKSTARTUP_RETRIES 3
  79
  80 /* maximum number of reset retries before giving up */
  81 #define MAX_HOST_RESET_RETRIES 5
  82
  83 /* Maximum number of error handler retries before giving up */
  84 #define MAX_ERR_HANDLER_RETRIES 5
  85
  86 /* Expose the flag value from utp_upiu_query.value */
  87 #define MASK_QUERY_UPIU_FLAG_LOC 0xFF
  88
  89 /* Interrupt aggregation default timeout, unit: 40us */
  90 #define INT_AGGR_DEF_TO 0x02
  91
  92 /* default delay of autosuspend: 2000 ms */
  93 #define RPM_AUTOSUSPEND_DELAY_MS 2000
  94
  95 /* Default delay of RPM device flush delayed work */
  96 #define RPM_DEV_FLUSH_RECHECK_WORK_DELAY_MS 5000
  97
  98 /* Default value of wait time before gating device ref clock */
  99 #define UFSHCD_REF_CLK_GATING_WAIT_US 0xFF /* microsecs */
 100
 101 /* Polling time to wait for fDeviceInit */
 102 #define FDEVICEINIT_COMPL_TIMEOUT 1500 /* millisecs */
 103
 104 /* Default RTC update every 10 seconds */
 105 #define UFS_RTC_UPDATE_INTERVAL_MS (10 * MSEC_PER_SEC)
 106
 107 /* bMaxNumOfRTT is equal to two after device manufacturing */
 108 #define DEFAULT_MAX_NUM_RTT 2
 109
 110 /* UFSHC 4.0 compliant HC support this mode. */
 111 static bool use_mcq_mode = true;
 112
 113 static bool is_mcq_supported(struct ufs_hba *hba)
 114 {
 115         return hba->mcq_sup && use_mcq_mode;
 116 }
 117
 118 module_param(use_mcq_mode, bool, 0644);
 119 MODULE_PARM_DESC(use_mcq_mode, "Control MCQ mode for controllers starting from UFSHCI 4.0. 1 - enable MCQ, 0 - disable MCQ. MCQ is enabled by default");
 120
 121 static unsigned int uic_cmd_timeout = UIC_CMD_TIMEOUT_DEFAULT;
 122
 123 static int uic_cmd_timeout_set(const char *val, const struct kernel_param *kp)
 124 {
 125         return param_set_uint_minmax(val, kp, UIC_CMD_TIMEOUT_DEFAULT,
 126                                      UIC_CMD_TIMEOUT_MAX);
 127 }
 128
 129 static const struct kernel_param_ops uic_cmd_timeout_ops = {
 130         .set = uic_cmd_timeout_set,
 131         .get = param_get_uint,
 132 };
 133
 134 module_param_cb(uic_cmd_timeout, &uic_cmd_timeout_ops, &uic_cmd_timeout, 0644);
 135 MODULE_PARM_DESC(uic_cmd_timeout,
 136                  "UFS UIC command timeout in milliseconds. Defaults to 500ms. Supported values range from 500ms to 2 seconds inclusively");
 137
 138 #define ufshcd_toggle_vreg(_dev, _vreg, _on)                            \
 139         ({                                                              \
 140                 int _ret;                                               \
 141                 if (_on)                                                \
 142                         _ret = ufshcd_enable_vreg(_dev, _vreg);         \
 143                 else                                                    \
 144                         _ret = ufshcd_disable_vreg(_dev, _vreg);        \
 145                 _ret;                                                   \
 146         })
 147
 148 #define ufshcd_hex_dump(prefix_str, buf, len) do {                       \
 149         size_t __len = (len);                                            \
 150         print_hex_dump(KERN_ERR, prefix_str,                             \
 151                        __len > 4 ? DUMP_PREFIX_OFFSET : DUMP_PREFIX_NONE,\
 152                        16, 4, buf, __len, false);                        \
 153 } while (0)
 154
 155 int ufshcd_dump_regs(struct ufs_hba *hba, size_t offset, size_t len,
 156                      const char *prefix)
 157 {
 158         u32 *regs;
 159         size_t pos;
 160
 161         if (offset % 4 != 0 || len % 4 != 0) /* keep readl happy */
 162                 return -EINVAL;
 163
 164         regs = kzalloc(len, GFP_ATOMIC);
 165         if (!regs)
 166                 return -ENOMEM;
 167
 168         for (pos = 0; pos < len; pos += 4) {
 169                 if (offset == 0 &&
 170                     pos >= REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER &&
 171                     pos <= REG_UIC_ERROR_CODE_DME)
 172                         continue;
 173                 regs[pos / 4] = ufshcd_readl(hba, offset + pos);
 174         }
 175
 176         ufshcd_hex_dump(prefix, regs, len);
 177         kfree(regs);
 178
 179         return 0;
 180 }
 181 EXPORT_SYMBOL_GPL(ufshcd_dump_regs);
 182
 183 enum {
 184         UFSHCD_MAX_CHANNEL      = 0,
 185         UFSHCD_MAX_ID           = 1,
 186 };
 187
 188 static const char *const ufshcd_state_name[] = {
 189         [UFSHCD_STATE_RESET]                    = "reset",
 190         [UFSHCD_STATE_OPERATIONAL]              = "operational",
 191         [UFSHCD_STATE_ERROR]                    = "error",
 192         [UFSHCD_STATE_EH_SCHEDULED_FATAL]       = "eh_fatal",
 193         [UFSHCD_STATE_EH_SCHEDULED_NON_FATAL]   = "eh_non_fatal",
 194 };
 195
 196 /* UFSHCD error handling flags */
 197 enum {
 198         UFSHCD_EH_IN_PROGRESS = (1 << 0),
 199 };
 200
 201 /* UFSHCD UIC layer error flags */
 202 enum {
 203         UFSHCD_UIC_DL_PA_INIT_ERROR = (1 << 0), /* Data link layer error */
 204         UFSHCD_UIC_DL_NAC_RECEIVED_ERROR = (1 << 1), /* Data link layer error */
 205         UFSHCD_UIC_DL_TCx_REPLAY_ERROR = (1 << 2), /* Data link layer error */
 206         UFSHCD_UIC_NL_ERROR = (1 << 3), /* Network layer error */
 207         UFSHCD_UIC_TL_ERROR = (1 << 4), /* Transport Layer error */
 208         UFSHCD_UIC_DME_ERROR = (1 << 5), /* DME error */
 209         UFSHCD_UIC_PA_GENERIC_ERROR = (1 << 6), /* Generic PA error */
 210 };
 211
 212 #define ufshcd_set_eh_in_progress(h) \
 213         ((h)->eh_flags |= UFSHCD_EH_IN_PROGRESS)
 214 #define ufshcd_eh_in_progress(h) \
 215         ((h)->eh_flags & UFSHCD_EH_IN_PROGRESS)
 216 #define ufshcd_clear_eh_in_progress(h) \
 217         ((h)->eh_flags &= ~UFSHCD_EH_IN_PROGRESS)
 218
 219 const struct ufs_pm_lvl_states ufs_pm_lvl_states[] = {
 220         [UFS_PM_LVL_0] = {UFS_ACTIVE_PWR_MODE, UIC_LINK_ACTIVE_STATE},
 221         [UFS_PM_LVL_1] = {UFS_ACTIVE_PWR_MODE, UIC_LINK_HIBERN8_STATE},
 222         [UFS_PM_LVL_2] = {UFS_SLEEP_PWR_MODE, UIC_LINK_ACTIVE_STATE},
 223         [UFS_PM_LVL_3] = {UFS_SLEEP_PWR_MODE, UIC_LINK_HIBERN8_STATE},
 224         [UFS_PM_LVL_4] = {UFS_POWERDOWN_PWR_MODE, UIC_LINK_HIBERN8_STATE},
 225         [UFS_PM_LVL_5] = {UFS_POWERDOWN_PWR_MODE, UIC_LINK_OFF_STATE},
 226         /*
 227          * For DeepSleep, the link is first put in hibern8 and then off.
 228          * Leaving the link in hibern8 is not supported.
 229          */
 230         [UFS_PM_LVL_6] = {UFS_DEEPSLEEP_PWR_MODE, UIC_LINK_OFF_STATE},
 231 };
 232
 233 static inline enum ufs_dev_pwr_mode
 234 ufs_get_pm_lvl_to_dev_pwr_mode(enum ufs_pm_level lvl)
 235 {
 236         return ufs_pm_lvl_states[lvl].dev_state;
 237 }
 238
 239 static inline enum uic_link_state
 240 ufs_get_pm_lvl_to_link_pwr_state(enum ufs_pm_level lvl)
 241 {
 242         return ufs_pm_lvl_states[lvl].link_state;
 243 }
 244
 245 static inline enum ufs_pm_level
 246 ufs_get_desired_pm_lvl_for_dev_link_state(enum ufs_dev_pwr_mode dev_state,
 247                                         enum uic_link_state link_state)
 248 {
 249         enum ufs_pm_level lvl;
 250
 251         for (lvl = UFS_PM_LVL_0; lvl < UFS_PM_LVL_MAX; lvl++) {
 252                 if ((ufs_pm_lvl_states[lvl].dev_state == dev_state) &&
 253                         (ufs_pm_lvl_states[lvl].link_state == link_state))
 254                         return lvl;
 255         }
 256
 257         /* if no match found, return the level 0 */
 258         return UFS_PM_LVL_0;
 259 }
 260
 261 static bool ufshcd_is_ufs_dev_busy(struct ufs_hba *hba)
 262 {
 263         return (hba->clk_gating.active_reqs || hba->outstanding_reqs || hba->outstanding_tasks ||
 264                 hba->active_uic_cmd || hba->uic_async_done);
 265 }
 266
 267 static const struct ufs_dev_quirk ufs_fixups[] = {
 268         /* UFS cards deviations table */
 269         { .wmanufacturerid = UFS_VENDOR_MICRON,
 270           .model = UFS_ANY_MODEL,
 271           .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM },
 272         { .wmanufacturerid = UFS_VENDOR_SAMSUNG,
 273           .model = UFS_ANY_MODEL,
 274           .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM |
 275                    UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE |
 276                    UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS },
 277         { .wmanufacturerid = UFS_VENDOR_SKHYNIX,
 278           .model = UFS_ANY_MODEL,
 279           .quirk = UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME },
 280         { .wmanufacturerid = UFS_VENDOR_SKHYNIX,
 281           .model = "hB8aL1" /*H28U62301AMR*/,
 282           .quirk = UFS_DEVICE_QUIRK_HOST_VS_DEBUGSAVECONFIGTIME },
 283         { .wmanufacturerid = UFS_VENDOR_TOSHIBA,
 284           .model = UFS_ANY_MODEL,
 285           .quirk = UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM },
 286         { .wmanufacturerid = UFS_VENDOR_TOSHIBA,
 287           .model = "THGLF2G9C8KBADG",
 288           .quirk = UFS_DEVICE_QUIRK_PA_TACTIVATE },
 289         { .wmanufacturerid = UFS_VENDOR_TOSHIBA,
 290           .model = "THGLF2G9D8KBADG",
 291           .quirk = UFS_DEVICE_QUIRK_PA_TACTIVATE },
 292         {}
 293 };
 294
 295 static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba);
 296 static void ufshcd_async_scan(void *data, async_cookie_t cookie);
 297 static int ufshcd_reset_and_restore(struct ufs_hba *hba);
 298 static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd);
 299 static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag);
 300 static void ufshcd_hba_exit(struct ufs_hba *hba);
 301 static int ufshcd_device_init(struct ufs_hba *hba, bool init_dev_params);
 302 static int ufshcd_probe_hba(struct ufs_hba *hba, bool init_dev_params);
 303 static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on);
 304 static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba);
 305 static int ufshcd_host_reset_and_restore(struct ufs_hba *hba);
 306 static void ufshcd_resume_clkscaling(struct ufs_hba *hba);
 307 static void ufshcd_suspend_clkscaling(struct ufs_hba *hba);
 308 static int ufshcd_scale_clks(struct ufs_hba *hba, unsigned long freq,
 309                              bool scale_up);
 310 static irqreturn_t ufshcd_intr(int irq, void *__hba);
 311 static int ufshcd_change_power_mode(struct ufs_hba *hba,
 312                              struct ufs_pa_layer_attr *pwr_mode);
 313 static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on);
 314 static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on);
 315 static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba,
 316                                          struct ufs_vreg *vreg);
 317 static void ufshcd_wb_toggle_buf_flush_during_h8(struct ufs_hba *hba,
 318                                                  bool enable);
 319 static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba);
 320 static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba);
 321
 322 void ufshcd_enable_irq(struct ufs_hba *hba)
 323 {
 324         if (!hba->is_irq_enabled) {
 325                 enable_irq(hba->irq);
 326                 hba->is_irq_enabled = true;
 327         }
 328 }
 329 EXPORT_SYMBOL_GPL(ufshcd_enable_irq);
 330
 331 void ufshcd_disable_irq(struct ufs_hba *hba)
 332 {
 333         if (hba->is_irq_enabled) {
 334                 disable_irq(hba->irq);
 335                 hba->is_irq_enabled = false;
 336         }
 337 }
 338 EXPORT_SYMBOL_GPL(ufshcd_disable_irq);
 339
 340 static void ufshcd_configure_wb(struct ufs_hba *hba)
 341 {
 342         if (!ufshcd_is_wb_allowed(hba))
 343                 return;
 344
 345         ufshcd_wb_toggle(hba, true);
 346
 347         ufshcd_wb_toggle_buf_flush_during_h8(hba, true);
 348
 349         if (ufshcd_is_wb_buf_flush_allowed(hba))
 350                 ufshcd_wb_toggle_buf_flush(hba, true);
 351 }
 352
 353 static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba, unsigned int tag,
 354                                       enum ufs_trace_str_t str_t)
 355 {
 356         struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr;
 357         struct utp_upiu_header *header;
 358
 359         if (!trace_ufshcd_upiu_enabled())
 360                 return;
 361
 362         if (str_t == UFS_CMD_SEND)
 363                 header = &rq->header;
 364         else
 365                 header = &hba->lrb[tag].ucd_rsp_ptr->header;
 366
 367         trace_ufshcd_upiu(dev_name(hba->dev), str_t, header, &rq->sc.cdb,
 368                           UFS_TSF_CDB);
 369 }
 370
 371 static void ufshcd_add_query_upiu_trace(struct ufs_hba *hba,
 372                                         enum ufs_trace_str_t str_t,
 373                                         struct utp_upiu_req *rq_rsp)
 374 {
 375         if (!trace_ufshcd_upiu_enabled())
 376                 return;
 377
 378         trace_ufshcd_upiu(dev_name(hba->dev), str_t, &rq_rsp->header,
 379                           &rq_rsp->qr, UFS_TSF_OSF);
 380 }
 381
 382 static void ufshcd_add_tm_upiu_trace(struct ufs_hba *hba, unsigned int tag,
 383                                      enum ufs_trace_str_t str_t)
 384 {
 385         struct utp_task_req_desc *descp = &hba->utmrdl_base_addr[tag];
 386
 387         if (!trace_ufshcd_upiu_enabled())
 388                 return;
 389
 390         if (str_t == UFS_TM_SEND)
 391                 trace_ufshcd_upiu(dev_name(hba->dev), str_t,
 392                                   &descp->upiu_req.req_header,
 393                                   &descp->upiu_req.input_param1,
 394                                   UFS_TSF_TM_INPUT);
 395         else
 396                 trace_ufshcd_upiu(dev_name(hba->dev), str_t,
 397                                   &descp->upiu_rsp.rsp_header,
 398                                   &descp->upiu_rsp.output_param1,
 399                                   UFS_TSF_TM_OUTPUT);
 400 }
 401
 402 static void ufshcd_add_uic_command_trace(struct ufs_hba *hba,
 403                                          const struct uic_command *ucmd,
 404                                          enum ufs_trace_str_t str_t)
 405 {
 406         u32 cmd;
 407
 408         if (!trace_ufshcd_uic_command_enabled())
 409                 return;
 410
 411         if (str_t == UFS_CMD_SEND)
 412                 cmd = ucmd->command;
 413         else
 414                 cmd = ufshcd_readl(hba, REG_UIC_COMMAND);
 415
 416         trace_ufshcd_uic_command(dev_name(hba->dev), str_t, cmd,
 417                                  ufshcd_readl(hba, REG_UIC_COMMAND_ARG_1),
 418                                  ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2),
 419                                  ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3));
 420 }
 421
 422 static void ufshcd_add_command_trace(struct ufs_hba *hba, unsigned int tag,
 423                                      enum ufs_trace_str_t str_t)
 424 {
 425         u64 lba = 0;
 426         u8 opcode = 0, group_id = 0;
 427         u32 doorbell = 0;
 428         u32 intr;
 429         int hwq_id = -1;
 430         struct ufshcd_lrb *lrbp = &hba->lrb[tag];
 431         struct scsi_cmnd *cmd = lrbp->cmd;
 432         struct request *rq = scsi_cmd_to_rq(cmd);
 433         int transfer_len = -1;
 434
 435         if (!cmd)
 436                 return;
 437
 438         /* trace UPIU also */
 439         ufshcd_add_cmd_upiu_trace(hba, tag, str_t);
 440         if (!trace_ufshcd_command_enabled())
 441                 return;
 442
 443         opcode = cmd->cmnd[0];
 444
 445         if (opcode == READ_10 || opcode == WRITE_10) {
 446                 /*
 447                  * Currently we only fully trace read(10) and write(10) commands
 448                  */
 449                 transfer_len =
 450                        be32_to_cpu(lrbp->ucd_req_ptr->sc.exp_data_transfer_len);
 451                 lba = scsi_get_lba(cmd);
 452                 if (opcode == WRITE_10)
 453                         group_id = lrbp->cmd->cmnd[6];
 454         } else if (opcode == UNMAP) {
 455                 /*
 456                  * The number of Bytes to be unmapped beginning with the lba.
 457                  */
 458                 transfer_len = blk_rq_bytes(rq);
 459                 lba = scsi_get_lba(cmd);
 460         }
 461
 462         intr = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
 463
 464         if (hba->mcq_enabled) {
 465                 struct ufs_hw_queue *hwq = ufshcd_mcq_req_to_hwq(hba, rq);
 466
 467                 hwq_id = hwq->id;
 468         } else {
 469                 doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
 470         }
 471         trace_ufshcd_command(cmd->device, str_t, tag, doorbell, hwq_id,
 472                              transfer_len, intr, lba, opcode, group_id);
 473 }
 474
 475 static void ufshcd_print_clk_freqs(struct ufs_hba *hba)
 476 {
 477         struct ufs_clk_info *clki;
 478         struct list_head *head = &hba->clk_list_head;
 479
 480         if (list_empty(head))
 481                 return;
 482
 483         list_for_each_entry(clki, head, list) {
 484                 if (!IS_ERR_OR_NULL(clki->clk) && clki->min_freq &&
 485                                 clki->max_freq)
 486                         dev_err(hba->dev, "clk: %s, rate: %u\n",
 487                                         clki->name, clki->curr_freq);
 488         }
 489 }
 490
 491 static void ufshcd_print_evt(struct ufs_hba *hba, u32 id,
 492                              const char *err_name)
 493 {
 494         int i;
 495         bool found = false;
 496         const struct ufs_event_hist *e;
 497
 498         if (id >= UFS_EVT_CNT)
 499                 return;
 500
 501         e = &hba->ufs_stats.event[id];
 502
 503         for (i = 0; i < UFS_EVENT_HIST_LENGTH; i++) {
 504                 int p = (i + e->pos) % UFS_EVENT_HIST_LENGTH;
 505
 506                 if (e->tstamp[p] == 0)
 507                         continue;
 508                 dev_err(hba->dev, "%s[%d] = 0x%x at %lld us\n", err_name, p,
 509                         e->val[p], div_u64(e->tstamp[p], 1000));
 510                 found = true;
 511         }
 512
 513         if (!found)
 514                 dev_err(hba->dev, "No record of %s\n", err_name);
 515         else
 516                 dev_err(hba->dev, "%s: total cnt=%llu\n", err_name, e->cnt);
 517 }
 518
 519 static void ufshcd_print_evt_hist(struct ufs_hba *hba)
 520 {
 521         ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
 522
 523         ufshcd_print_evt(hba, UFS_EVT_PA_ERR, "pa_err");
 524         ufshcd_print_evt(hba, UFS_EVT_DL_ERR, "dl_err");
 525         ufshcd_print_evt(hba, UFS_EVT_NL_ERR, "nl_err");
 526         ufshcd_print_evt(hba, UFS_EVT_TL_ERR, "tl_err");
 527         ufshcd_print_evt(hba, UFS_EVT_DME_ERR, "dme_err");
 528         ufshcd_print_evt(hba, UFS_EVT_AUTO_HIBERN8_ERR,
 529                          "auto_hibern8_err");
 530         ufshcd_print_evt(hba, UFS_EVT_FATAL_ERR, "fatal_err");
 531         ufshcd_print_evt(hba, UFS_EVT_LINK_STARTUP_FAIL,
 532                          "link_startup_fail");
 533         ufshcd_print_evt(hba, UFS_EVT_RESUME_ERR, "resume_fail");
 534         ufshcd_print_evt(hba, UFS_EVT_SUSPEND_ERR,
 535                          "suspend_fail");
 536         ufshcd_print_evt(hba, UFS_EVT_WL_RES_ERR, "wlun resume_fail");
 537         ufshcd_print_evt(hba, UFS_EVT_WL_SUSP_ERR,
 538                          "wlun suspend_fail");
 539         ufshcd_print_evt(hba, UFS_EVT_DEV_RESET, "dev_reset");
 540         ufshcd_print_evt(hba, UFS_EVT_HOST_RESET, "host_reset");
 541         ufshcd_print_evt(hba, UFS_EVT_ABORT, "task_abort");
 542
 543         ufshcd_vops_dbg_register_dump(hba);
 544 }
 545
 546 static
 547 void ufshcd_print_tr(struct ufs_hba *hba, int tag, bool pr_prdt)
 548 {
 549         const struct ufshcd_lrb *lrbp;
 550         int prdt_length;
 551
 552         lrbp = &hba->lrb[tag];
 553
 554         dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n",
 555                         tag, div_u64(lrbp->issue_time_stamp_local_clock, 1000));
 556         dev_err(hba->dev, "UPIU[%d] - complete time %lld us\n",
 557                         tag, div_u64(lrbp->compl_time_stamp_local_clock, 1000));
 558         dev_err(hba->dev,
 559                 "UPIU[%d] - Transfer Request Descriptor phys@0x%llx\n",
 560                 tag, (u64)lrbp->utrd_dma_addr);
 561
 562         ufshcd_hex_dump("UPIU TRD: ", lrbp->utr_descriptor_ptr,
 563                         sizeof(struct utp_transfer_req_desc));
 564         dev_err(hba->dev, "UPIU[%d] - Request UPIU phys@0x%llx\n", tag,
 565                 (u64)lrbp->ucd_req_dma_addr);
 566         ufshcd_hex_dump("UPIU REQ: ", lrbp->ucd_req_ptr,
 567                         sizeof(struct utp_upiu_req));
 568         dev_err(hba->dev, "UPIU[%d] - Response UPIU phys@0x%llx\n", tag,
 569                 (u64)lrbp->ucd_rsp_dma_addr);
 570         ufshcd_hex_dump("UPIU RSP: ", lrbp->ucd_rsp_ptr,
 571                         sizeof(struct utp_upiu_rsp));
 572
 573         prdt_length = le16_to_cpu(
 574                 lrbp->utr_descriptor_ptr->prd_table_length);
 575         if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
 576                 prdt_length /= ufshcd_sg_entry_size(hba);
 577
 578         dev_err(hba->dev,
 579                 "UPIU[%d] - PRDT - %d entries  phys@0x%llx\n",
 580                 tag, prdt_length,
 581                 (u64)lrbp->ucd_prdt_dma_addr);
 582
 583         if (pr_prdt)
 584                 ufshcd_hex_dump("UPIU PRDT: ", lrbp->ucd_prdt_ptr,
 585                         ufshcd_sg_entry_size(hba) * prdt_length);
 586 }
 587
 588 static bool ufshcd_print_tr_iter(struct request *req, void *priv)
 589 {
 590         struct scsi_device *sdev = req->q->queuedata;
 591         struct Scsi_Host *shost = sdev->host;
 592         struct ufs_hba *hba = shost_priv(shost);
 593
 594         ufshcd_print_tr(hba, req->tag, *(bool *)priv);
 595
 596         return true;
 597 }
 598
 599 /**
 600  * ufshcd_print_trs_all - print trs for all started requests.
 601  * @hba: per-adapter instance.
 602  * @pr_prdt: need to print prdt or not.
 603  */
 604 static void ufshcd_print_trs_all(struct ufs_hba *hba, bool pr_prdt)
 605 {
 606         blk_mq_tagset_busy_iter(&hba->host->tag_set, ufshcd_print_tr_iter, &pr_prdt);
 607 }
 608
 609 static void ufshcd_print_tmrs(struct ufs_hba *hba, unsigned long bitmap)
 610 {
 611         int tag;
 612
 613         for_each_set_bit(tag, &bitmap, hba->nutmrs) {
 614                 struct utp_task_req_desc *tmrdp = &hba->utmrdl_base_addr[tag];
 615
 616                 dev_err(hba->dev, "TM[%d] - Task Management Header\n", tag);
 617                 ufshcd_hex_dump("", tmrdp, sizeof(*tmrdp));
 618         }
 619 }
 620
 621 static void ufshcd_print_host_state(struct ufs_hba *hba)
 622 {
 623         const struct scsi_device *sdev_ufs = hba->ufs_device_wlun;
 624
 625         dev_err(hba->dev, "UFS Host state=%d\n", hba->ufshcd_state);
 626         dev_err(hba->dev, "outstanding reqs=0x%lx tasks=0x%lx\n",
 627                 hba->outstanding_reqs, hba->outstanding_tasks);
 628         dev_err(hba->dev, "saved_err=0x%x, saved_uic_err=0x%x\n",
 629                 hba->saved_err, hba->saved_uic_err);
 630         dev_err(hba->dev, "Device power mode=%d, UIC link state=%d\n",
 631                 hba->curr_dev_pwr_mode, hba->uic_link_state);
 632         dev_err(hba->dev, "PM in progress=%d, sys. suspended=%d\n",
 633                 hba->pm_op_in_progress, hba->is_sys_suspended);
 634         dev_err(hba->dev, "Auto BKOPS=%d, Host self-block=%d\n",
 635                 hba->auto_bkops_enabled, hba->host->host_self_blocked);
 636         dev_err(hba->dev, "Clk gate=%d\n", hba->clk_gating.state);
 637         dev_err(hba->dev,
 638                 "last_hibern8_exit_tstamp at %lld us, hibern8_exit_cnt=%d\n",
 639                 div_u64(hba->ufs_stats.last_hibern8_exit_tstamp, 1000),
 640                 hba->ufs_stats.hibern8_exit_cnt);
 641         dev_err(hba->dev, "last intr at %lld us, last intr status=0x%x\n",
 642                 div_u64(hba->ufs_stats.last_intr_ts, 1000),
 643                 hba->ufs_stats.last_intr_status);
 644         dev_err(hba->dev, "error handling flags=0x%x, req. abort count=%d\n",
 645                 hba->eh_flags, hba->req_abort_count);
 646         dev_err(hba->dev, "hba->ufs_version=0x%x, Host capabilities=0x%x, caps=0x%x\n",
 647                 hba->ufs_version, hba->capabilities, hba->caps);
 648         dev_err(hba->dev, "quirks=0x%x, dev. quirks=0x%x\n", hba->quirks,
 649                 hba->dev_quirks);
 650         if (sdev_ufs)
 651                 dev_err(hba->dev, "UFS dev info: %.8s %.16s rev %.4s\n",
 652                         sdev_ufs->vendor, sdev_ufs->model, sdev_ufs->rev);
 653
 654         ufshcd_print_clk_freqs(hba);
 655 }
 656
 657 /**
 658  * ufshcd_print_pwr_info - print power params as saved in hba
 659  * power info
 660  * @hba: per-adapter instance
 661  */
 662 static void ufshcd_print_pwr_info(struct ufs_hba *hba)
 663 {
 664         static const char * const names[] = {
 665                 "INVALID MODE",
 666                 "FAST MODE",
 667                 "SLOW_MODE",
 668                 "INVALID MODE",
 669                 "FASTAUTO_MODE",
 670                 "SLOWAUTO_MODE",
 671                 "INVALID MODE",
 672         };
 673
 674         /*
 675          * Using dev_dbg to avoid messages during runtime PM to avoid
 676          * never-ending cycles of messages written back to storage by user space
 677          * causing runtime resume, causing more messages and so on.
 678          */
 679         dev_dbg(hba->dev, "%s:[RX, TX]: gear=[%d, %d], lane[%d, %d], pwr[%s, %s], rate = %d\n",
 680                  __func__,
 681                  hba->pwr_info.gear_rx, hba->pwr_info.gear_tx,
 682                  hba->pwr_info.lane_rx, hba->pwr_info.lane_tx,
 683                  names[hba->pwr_info.pwr_rx],
 684                  names[hba->pwr_info.pwr_tx],
 685                  hba->pwr_info.hs_rate);
 686 }
 687
 688 static void ufshcd_device_reset(struct ufs_hba *hba)
 689 {
 690         int err;
 691
 692         err = ufshcd_vops_device_reset(hba);
 693
 694         if (!err) {
 695                 ufshcd_set_ufs_dev_active(hba);
 696                 if (ufshcd_is_wb_allowed(hba)) {
 697                         hba->dev_info.wb_enabled = false;
 698                         hba->dev_info.wb_buf_flush_enabled = false;
 699                 }
 700                 if (hba->dev_info.rtc_type == UFS_RTC_RELATIVE)
 701                         hba->dev_info.rtc_time_baseline = 0;
 702         }
 703         if (err != -EOPNOTSUPP)
 704                 ufshcd_update_evt_hist(hba, UFS_EVT_DEV_RESET, err);
 705 }
 706
 707 void ufshcd_delay_us(unsigned long us, unsigned long tolerance)
 708 {
 709         if (!us)
 710                 return;
 711
 712         if (us < 10)
 713                 udelay(us);
 714         else
 715                 usleep_range(us, us + tolerance);
 716 }
 717 EXPORT_SYMBOL_GPL(ufshcd_delay_us);
 718
 719 /**
 720  * ufshcd_wait_for_register - wait for register value to change
 721  * @hba: per-adapter interface
 722  * @reg: mmio register offset
 723  * @mask: mask to apply to the read register value
 724  * @val: value to wait for
 725  * @interval_us: polling interval in microseconds
 726  * @timeout_ms: timeout in milliseconds
 727  *
 728  * Return: -ETIMEDOUT on error, zero on success.
 729  */
 730 static int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask,
 731                                     u32 val, unsigned long interval_us,
 732                                     unsigned long timeout_ms)
 733 {
 734         u32 v;
 735
 736         val &= mask; /* ignore bits that we don't intend to wait on */
 737
 738         return read_poll_timeout(ufshcd_readl, v, (v & mask) == val,
 739                                  interval_us, timeout_ms * 1000, false, hba, reg);
 740 }
 741
 742 /**
 743  * ufshcd_get_intr_mask - Get the interrupt bit mask
 744  * @hba: Pointer to adapter instance
 745  *
 746  * Return: interrupt bit mask per version
 747  */
 748 static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba)
 749 {
 750         if (hba->ufs_version <= ufshci_version(2, 0))
 751                 return INTERRUPT_MASK_ALL_VER_11;
 752
 753         return INTERRUPT_MASK_ALL_VER_21;
 754 }
 755
 756 /**
 757  * ufshcd_get_ufs_version - Get the UFS version supported by the HBA
 758  * @hba: Pointer to adapter instance
 759  *
 760  * Return: UFSHCI version supported by the controller
 761  */
 762 static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba)
 763 {
 764         u32 ufshci_ver;
 765
 766         if (hba->quirks & UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION)
 767                 ufshci_ver = ufshcd_vops_get_ufs_hci_version(hba);
 768         else
 769                 ufshci_ver = ufshcd_readl(hba, REG_UFS_VERSION);
 770
 771         /*
 772          * UFSHCI v1.x uses a different version scheme, in order
 773          * to allow the use of comparisons with the ufshci_version
 774          * function, we convert it to the same scheme as ufs 2.0+.
 775          */
 776         if (ufshci_ver & 0x00010000)
 777                 return ufshci_version(1, ufshci_ver & 0x00000100);
 778
 779         return ufshci_ver;
 780 }
 781
 782 /**
 783  * ufshcd_is_device_present - Check if any device connected to
 784  *                            the host controller
 785  * @hba: pointer to adapter instance
 786  *
 787  * Return: true if device present, false if no device detected
 788  */
 789 static inline bool ufshcd_is_device_present(struct ufs_hba *hba)
 790 {
 791         return ufshcd_readl(hba, REG_CONTROLLER_STATUS) & DEVICE_PRESENT;
 792 }
 793
 794 /**
 795  * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status
 796  * @lrbp: pointer to local command reference block
 797  * @cqe: pointer to the completion queue entry
 798  *
 799  * This function is used to get the OCS field from UTRD
 800  *
 801  * Return: the OCS field in the UTRD.
 802  */
 803 static enum utp_ocs ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp,
 804                                       struct cq_entry *cqe)
 805 {
 806         if (cqe)
 807                 return le32_to_cpu(cqe->status) & MASK_OCS;
 808
 809         return lrbp->utr_descriptor_ptr->header.ocs & MASK_OCS;
 810 }
 811
 812 /**
 813  * ufshcd_utrl_clear() - Clear requests from the controller request list.
 814  * @hba: per adapter instance
 815  * @mask: mask with one bit set for each request to be cleared
 816  */
 817 static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 mask)
 818 {
 819         if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
 820                 mask = ~mask;
 821         /*
 822          * From the UFSHCI specification: "UTP Transfer Request List CLear
 823          * Register (UTRLCLR): This field is bit significant. Each bit
 824          * corresponds to a slot in the UTP Transfer Request List, where bit 0
 825          * corresponds to request slot 0. A bit in this field is set to ‘0’
 826          * by host software to indicate to the host controller that a transfer
 827          * request slot is cleared. The host controller
 828          * shall free up any resources associated to the request slot
 829          * immediately, and shall set the associated bit in UTRLDBR to ‘0’. The
 830          * host software indicates no change to request slots by setting the
 831          * associated bits in this field to ‘1’. Bits in this field shall only
 832          * be set ‘1’ or ‘0’ by host software when UTRLRSR is set to ‘1’."
 833          */
 834         ufshcd_writel(hba, ~mask, REG_UTP_TRANSFER_REQ_LIST_CLEAR);
 835 }
 836
 837 /**
 838  * ufshcd_utmrl_clear - Clear a bit in UTMRLCLR register
 839  * @hba: per adapter instance
 840  * @pos: position of the bit to be cleared
 841  */
 842 static inline void ufshcd_utmrl_clear(struct ufs_hba *hba, u32 pos)
 843 {
 844         if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
 845                 ufshcd_writel(hba, (1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
 846         else
 847                 ufshcd_writel(hba, ~(1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
 848 }
 849
 850 /**
 851  * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY
 852  * @reg: Register value of host controller status
 853  *
 854  * Return: 0 on success; a positive value if failed.
 855  */
 856 static inline int ufshcd_get_lists_status(u32 reg)
 857 {
 858         return !((reg & UFSHCD_STATUS_READY) == UFSHCD_STATUS_READY);
 859 }
 860
 861 /**
 862  * ufshcd_get_uic_cmd_result - Get the UIC command result
 863  * @hba: Pointer to adapter instance
 864  *
 865  * This function gets the result of UIC command completion
 866  *
 867  * Return: 0 on success; non-zero value on error.
 868  */
 869 static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba)
 870 {
 871         return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2) &
 872                MASK_UIC_COMMAND_RESULT;
 873 }
 874
 875 /**
 876  * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command
 877  * @hba: Pointer to adapter instance
 878  *
 879  * This function gets UIC command argument3
 880  *
 881  * Return: 0 on success; non-zero value on error.
 882  */
 883 static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba)
 884 {
 885         return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3);
 886 }
 887
 888 /**
 889  * ufshcd_get_req_rsp - returns the TR response transaction type
 890  * @ucd_rsp_ptr: pointer to response UPIU
 891  *
 892  * Return: UPIU type.
 893  */
 894 static inline enum upiu_response_transaction
 895 ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr)
 896 {
 897         return ucd_rsp_ptr->header.transaction_code;
 898 }
 899
 900 /**
 901  * ufshcd_is_exception_event - Check if the device raised an exception event
 902  * @ucd_rsp_ptr: pointer to response UPIU
 903  *
 904  * The function checks if the device raised an exception event indicated in
 905  * the Device Information field of response UPIU.
 906  *
 907  * Return: true if exception is raised, false otherwise.
 908  */
 909 static inline bool ufshcd_is_exception_event(struct utp_upiu_rsp *ucd_rsp_ptr)
 910 {
 911         return ucd_rsp_ptr->header.device_information & 1;
 912 }
 913
 914 /**
 915  * ufshcd_reset_intr_aggr - Reset interrupt aggregation values.
 916  * @hba: per adapter instance
 917  */
 918 static inline void
 919 ufshcd_reset_intr_aggr(struct ufs_hba *hba)
 920 {
 921         ufshcd_writel(hba, INT_AGGR_ENABLE |
 922                       INT_AGGR_COUNTER_AND_TIMER_RESET,
 923                       REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
 924 }
 925
 926 /**
 927  * ufshcd_config_intr_aggr - Configure interrupt aggregation values.
 928  * @hba: per adapter instance
 929  * @cnt: Interrupt aggregation counter threshold
 930  * @tmout: Interrupt aggregation timeout value
 931  */
 932 static inline void
 933 ufshcd_config_intr_aggr(struct ufs_hba *hba, u8 cnt, u8 tmout)
 934 {
 935         ufshcd_writel(hba, INT_AGGR_ENABLE | INT_AGGR_PARAM_WRITE |
 936                       INT_AGGR_COUNTER_THLD_VAL(cnt) |
 937                       INT_AGGR_TIMEOUT_VAL(tmout),
 938                       REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
 939 }
 940
 941 /**
 942  * ufshcd_disable_intr_aggr - Disables interrupt aggregation.
 943  * @hba: per adapter instance
 944  */
 945 static inline void ufshcd_disable_intr_aggr(struct ufs_hba *hba)
 946 {
 947         ufshcd_writel(hba, 0, REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
 948 }
 949
 950 /**
 951  * ufshcd_enable_run_stop_reg - Enable run-stop registers,
 952  *                      When run-stop registers are set to 1, it indicates the
 953  *                      host controller that it can process the requests
 954  * @hba: per adapter instance
 955  */
 956 static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba)
 957 {
 958         ufshcd_writel(hba, UTP_TASK_REQ_LIST_RUN_STOP_BIT,
 959                       REG_UTP_TASK_REQ_LIST_RUN_STOP);
 960         ufshcd_writel(hba, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT,
 961                       REG_UTP_TRANSFER_REQ_LIST_RUN_STOP);
 962 }
 963
 964 /**
 965  * ufshcd_hba_start - Start controller initialization sequence
 966  * @hba: per adapter instance
 967  */
 968 static inline void ufshcd_hba_start(struct ufs_hba *hba)
 969 {
 970         u32 val = CONTROLLER_ENABLE;
 971
 972         if (ufshcd_crypto_enable(hba))
 973                 val |= CRYPTO_GENERAL_ENABLE;
 974
 975         ufshcd_writel(hba, val, REG_CONTROLLER_ENABLE);
 976 }
 977
 978 /**
 979  * ufshcd_is_hba_active - Get controller state
 980  * @hba: per adapter instance
 981  *
 982  * Return: true if and only if the controller is active.
 983  */
 984 bool ufshcd_is_hba_active(struct ufs_hba *hba)
 985 {
 986         return ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & CONTROLLER_ENABLE;
 987 }
 988 EXPORT_SYMBOL_GPL(ufshcd_is_hba_active);
 989
 990 /**
 991  * ufshcd_pm_qos_init - initialize PM QoS request
 992  * @hba: per adapter instance
 993  */
 994 void ufshcd_pm_qos_init(struct ufs_hba *hba)
 995 {
 996
 997         if (hba->pm_qos_enabled)
 998                 return;
 999
1000         cpu_latency_qos_add_request(&hba->pm_qos_req, PM_QOS_DEFAULT_VALUE);
1001
1002         if (cpu_latency_qos_request_active(&hba->pm_qos_req))
1003                 hba->pm_qos_enabled = true;
1004 }
1005
1006 /**
1007  * ufshcd_pm_qos_exit - remove request from PM QoS
1008  * @hba: per adapter instance
1009  */
1010 void ufshcd_pm_qos_exit(struct ufs_hba *hba)
1011 {
1012         if (!hba->pm_qos_enabled)
1013                 return;
1014
1015         cpu_latency_qos_remove_request(&hba->pm_qos_req);
1016         hba->pm_qos_enabled = false;
1017 }
1018
1019 /**
1020  * ufshcd_pm_qos_update - update PM QoS request
1021  * @hba: per adapter instance
1022  * @on: If True, vote for perf PM QoS mode otherwise power save mode
1023  */
1024 static void ufshcd_pm_qos_update(struct ufs_hba *hba, bool on)
1025 {
1026         if (!hba->pm_qos_enabled)
1027                 return;
1028
1029         cpu_latency_qos_update_request(&hba->pm_qos_req, on ? 0 : PM_QOS_DEFAULT_VALUE);
1030 }
1031
1032 /**
1033  * ufshcd_set_clk_freq - set UFS controller clock frequencies
1034  * @hba: per adapter instance
1035  * @scale_up: If True, set max possible frequency othewise set low frequency
1036  *
1037  * Return: 0 if successful; < 0 upon failure.
1038  */
1039 static int ufshcd_set_clk_freq(struct ufs_hba *hba, bool scale_up)
1040 {
1041         int ret = 0;
1042         struct ufs_clk_info *clki;
1043         struct list_head *head = &hba->clk_list_head;
1044
1045         if (list_empty(head))
1046                 goto out;
1047
1048         list_for_each_entry(clki, head, list) {
1049                 if (!IS_ERR_OR_NULL(clki->clk)) {
1050                         if (scale_up && clki->max_freq) {
1051                                 if (clki->curr_freq == clki->max_freq)
1052                                         continue;
1053
1054                                 ret = clk_set_rate(clki->clk, clki->max_freq);
1055                                 if (ret) {
1056                                         dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
1057                                                 __func__, clki->name,
1058                                                 clki->max_freq, ret);
1059                                         break;
1060                                 }
1061                                 trace_ufshcd_clk_scaling(dev_name(hba->dev),
1062                                                 "scaled up", clki->name,
1063                                                 clki->curr_freq,
1064                                                 clki->max_freq);
1065
1066                                 clki->curr_freq = clki->max_freq;
1067
1068                         } else if (!scale_up && clki->min_freq) {
1069                                 if (clki->curr_freq == clki->min_freq)
1070                                         continue;
1071
1072                                 ret = clk_set_rate(clki->clk, clki->min_freq);
1073                                 if (ret) {
1074                                         dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
1075                                                 __func__, clki->name,
1076                                                 clki->min_freq, ret);
1077                                         break;
1078                                 }
1079                                 trace_ufshcd_clk_scaling(dev_name(hba->dev),
1080                                                 "scaled down", clki->name,
1081                                                 clki->curr_freq,
1082                                                 clki->min_freq);
1083                                 clki->curr_freq = clki->min_freq;
1084                         }
1085                 }
1086                 dev_dbg(hba->dev, "%s: clk: %s, rate: %lu\n", __func__,
1087                                 clki->name, clk_get_rate(clki->clk));
1088         }
1089
1090 out:
1091         return ret;
1092 }
1093
1094 int ufshcd_opp_config_clks(struct device *dev, struct opp_table *opp_table,
1095                            struct dev_pm_opp *opp, void *data,
1096                            bool scaling_down)
1097 {
1098         struct ufs_hba *hba = dev_get_drvdata(dev);
1099         struct list_head *head = &hba->clk_list_head;
1100         struct ufs_clk_info *clki;
1101         unsigned long freq;
1102         u8 idx = 0;
1103         int ret;
1104
1105         list_for_each_entry(clki, head, list) {
1106                 if (!IS_ERR_OR_NULL(clki->clk)) {
1107                         freq = dev_pm_opp_get_freq_indexed(opp, idx++);
1108
1109                         /* Do not set rate for clocks having frequency as 0 */
1110                         if (!freq)
1111                                 continue;
1112
1113                         ret = clk_set_rate(clki->clk, freq);
1114                         if (ret) {
1115                                 dev_err(dev, "%s: %s clk set rate(%ldHz) failed, %d\n",
1116                                         __func__, clki->name, freq, ret);
1117                                 return ret;
1118                         }
1119
1120                         trace_ufshcd_clk_scaling(dev_name(dev),
1121                                 (scaling_down ? "scaled down" : "scaled up"),
1122                                 clki->name, hba->clk_scaling.target_freq, freq);
1123                 }
1124         }
1125
1126         return 0;
1127 }
1128 EXPORT_SYMBOL_GPL(ufshcd_opp_config_clks);
1129
1130 static int ufshcd_opp_set_rate(struct ufs_hba *hba, unsigned long freq)
1131 {
1132         struct dev_pm_opp *opp;
1133         int ret;
1134
1135         opp = dev_pm_opp_find_freq_floor_indexed(hba->dev,
1136                                                  &freq, 0);
1137         if (IS_ERR(opp))
1138                 return PTR_ERR(opp);
1139
1140         ret = dev_pm_opp_set_opp(hba->dev, opp);
1141         dev_pm_opp_put(opp);
1142
1143         return ret;
1144 }
1145
1146 /**
1147  * ufshcd_scale_clks - scale up or scale down UFS controller clocks
1148  * @hba: per adapter instance
1149  * @freq: frequency to scale
1150  * @scale_up: True if scaling up and false if scaling down
1151  *
1152  * Return: 0 if successful; < 0 upon failure.
1153  */
1154 static int ufshcd_scale_clks(struct ufs_hba *hba, unsigned long freq,
1155                              bool scale_up)
1156 {
1157         int ret = 0;
1158         ktime_t start = ktime_get();
1159
1160         ret = ufshcd_vops_clk_scale_notify(hba, scale_up, PRE_CHANGE);
1161         if (ret)
1162                 goto out;
1163
1164         if (hba->use_pm_opp)
1165                 ret = ufshcd_opp_set_rate(hba, freq);
1166         else
1167                 ret = ufshcd_set_clk_freq(hba, scale_up);
1168         if (ret)
1169                 goto out;
1170
1171         ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE);
1172         if (ret) {
1173                 if (hba->use_pm_opp)
1174                         ufshcd_opp_set_rate(hba,
1175                                             hba->devfreq->previous_freq);
1176                 else
1177                         ufshcd_set_clk_freq(hba, !scale_up);
1178                 goto out;
1179         }
1180
1181         ufshcd_pm_qos_update(hba, scale_up);
1182
1183 out:
1184         trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
1185                         (scale_up ? "up" : "down"),
1186                         ktime_to_us(ktime_sub(ktime_get(), start)), ret);
1187         return ret;
1188 }
1189
1190 /**
1191  * ufshcd_is_devfreq_scaling_required - check if scaling is required or not
1192  * @hba: per adapter instance
1193  * @freq: frequency to scale
1194  * @scale_up: True if scaling up and false if scaling down
1195  *
1196  * Return: true if scaling is required, false otherwise.
1197  */
1198 static bool ufshcd_is_devfreq_scaling_required(struct ufs_hba *hba,
1199                                                unsigned long freq, bool scale_up)
1200 {
1201         struct ufs_clk_info *clki;
1202         struct list_head *head = &hba->clk_list_head;
1203
1204         if (list_empty(head))
1205                 return false;
1206
1207         if (hba->use_pm_opp)
1208                 return freq != hba->clk_scaling.target_freq;
1209
1210         list_for_each_entry(clki, head, list) {
1211                 if (!IS_ERR_OR_NULL(clki->clk)) {
1212                         if (scale_up && clki->max_freq) {
1213                                 if (clki->curr_freq == clki->max_freq)
1214                                         continue;
1215                                 return true;
1216                         } else if (!scale_up && clki->min_freq) {
1217                                 if (clki->curr_freq == clki->min_freq)
1218                                         continue;
1219                                 return true;
1220                         }
1221                 }
1222         }
1223
1224         return false;
1225 }
1226
1227 /*
1228  * Determine the number of pending commands by counting the bits in the SCSI
1229  * device budget maps. This approach has been selected because a bit is set in
1230  * the budget map before scsi_host_queue_ready() checks the host_self_blocked
1231  * flag. The host_self_blocked flag can be modified by calling
1232  * scsi_block_requests() or scsi_unblock_requests().
1233  */
1234 static u32 ufshcd_pending_cmds(struct ufs_hba *hba)
1235 {
1236         const struct scsi_device *sdev;
1237         unsigned long flags;
1238         u32 pending = 0;
1239
1240         spin_lock_irqsave(hba->host->host_lock, flags);
1241         __shost_for_each_device(sdev, hba->host)
1242                 pending += sbitmap_weight(&sdev->budget_map);
1243         spin_unlock_irqrestore(hba->host->host_lock, flags);
1244
1245         return pending;
1246 }
1247
1248 /*
1249  * Wait until all pending SCSI commands and TMFs have finished or the timeout
1250  * has expired.
1251  *
1252  * Return: 0 upon success; -EBUSY upon timeout.
1253  */
1254 static int ufshcd_wait_for_doorbell_clr(struct ufs_hba *hba,
1255                                         u64 wait_timeout_us)
1256 {
1257         int ret = 0;
1258         u32 tm_doorbell;
1259         u32 tr_pending;
1260         bool timeout = false, do_last_check = false;
1261         ktime_t start;
1262
1263         ufshcd_hold(hba);
1264         /*
1265          * Wait for all the outstanding tasks/transfer requests.
1266          * Verify by checking the doorbell registers are clear.
1267          */
1268         start = ktime_get();
1269         do {
1270                 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) {
1271                         ret = -EBUSY;
1272                         goto out;
1273                 }
1274
1275                 tm_doorbell = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
1276                 tr_pending = ufshcd_pending_cmds(hba);
1277                 if (!tm_doorbell && !tr_pending) {
1278                         timeout = false;
1279                         break;
1280                 } else if (do_last_check) {
1281                         break;
1282                 }
1283
1284                 io_schedule_timeout(msecs_to_jiffies(20));
1285                 if (ktime_to_us(ktime_sub(ktime_get(), start)) >
1286                     wait_timeout_us) {
1287                         timeout = true;
1288                         /*
1289                          * We might have scheduled out for long time so make
1290                          * sure to check if doorbells are cleared by this time
1291                          * or not.
1292                          */
1293                         do_last_check = true;
1294                 }
1295         } while (tm_doorbell || tr_pending);
1296
1297         if (timeout) {
1298                 dev_err(hba->dev,
1299                         "%s: timedout waiting for doorbell to clear (tm=0x%x, tr=0x%x)\n",
1300                         __func__, tm_doorbell, tr_pending);
1301                 ret = -EBUSY;
1302         }
1303 out:
1304         ufshcd_release(hba);
1305         return ret;
1306 }
1307
1308 /**
1309  * ufshcd_scale_gear - scale up/down UFS gear
1310  * @hba: per adapter instance
1311  * @scale_up: True for scaling up gear and false for scaling down
1312  *
1313  * Return: 0 for success; -EBUSY if scaling can't happen at this time;
1314  * non-zero for any other errors.
1315  */
1316 static int ufshcd_scale_gear(struct ufs_hba *hba, bool scale_up)
1317 {
1318         int ret = 0;
1319         struct ufs_pa_layer_attr new_pwr_info;
1320
1321         if (scale_up) {
1322                 memcpy(&new_pwr_info, &hba->clk_scaling.saved_pwr_info,
1323                        sizeof(struct ufs_pa_layer_attr));
1324         } else {
1325                 memcpy(&new_pwr_info, &hba->pwr_info,
1326                        sizeof(struct ufs_pa_layer_attr));
1327
1328                 if (hba->pwr_info.gear_tx > hba->clk_scaling.min_gear ||
1329                     hba->pwr_info.gear_rx > hba->clk_scaling.min_gear) {
1330                         /* save the current power mode */
1331                         memcpy(&hba->clk_scaling.saved_pwr_info,
1332                                 &hba->pwr_info,
1333                                 sizeof(struct ufs_pa_layer_attr));
1334
1335                         /* scale down gear */
1336                         new_pwr_info.gear_tx = hba->clk_scaling.min_gear;
1337                         new_pwr_info.gear_rx = hba->clk_scaling.min_gear;
1338                 }
1339         }
1340
1341         /* check if the power mode needs to be changed or not? */
1342         ret = ufshcd_config_pwr_mode(hba, &new_pwr_info);
1343         if (ret)
1344                 dev_err(hba->dev, "%s: failed err %d, old gear: (tx %d rx %d), new gear: (tx %d rx %d)",
1345                         __func__, ret,
1346                         hba->pwr_info.gear_tx, hba->pwr_info.gear_rx,
1347                         new_pwr_info.gear_tx, new_pwr_info.gear_rx);
1348
1349         return ret;
1350 }
1351
1352 /*
1353  * Wait until all pending SCSI commands and TMFs have finished or the timeout
1354  * has expired.
1355  *
1356  * Return: 0 upon success; -EBUSY upon timeout.
1357  */
1358 static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba, u64 timeout_us)
1359 {
1360         int ret = 0;
1361         /*
1362          * make sure that there are no outstanding requests when
1363          * clock scaling is in progress
1364          */
1365         blk_mq_quiesce_tagset(&hba->host->tag_set);
1366         mutex_lock(&hba->wb_mutex);
1367         down_write(&hba->clk_scaling_lock);
1368
1369         if (!hba->clk_scaling.is_allowed ||
1370             ufshcd_wait_for_doorbell_clr(hba, timeout_us)) {
1371                 ret = -EBUSY;
1372                 up_write(&hba->clk_scaling_lock);
1373                 mutex_unlock(&hba->wb_mutex);
1374                 blk_mq_unquiesce_tagset(&hba->host->tag_set);
1375                 goto out;
1376         }
1377
1378         /* let's not get into low power until clock scaling is completed */
1379         ufshcd_hold(hba);
1380
1381 out:
1382         return ret;
1383 }
1384
1385 static void ufshcd_clock_scaling_unprepare(struct ufs_hba *hba, int err, bool scale_up)
1386 {
1387         up_write(&hba->clk_scaling_lock);
1388
1389         /* Enable Write Booster if we have scaled up else disable it */
1390         if (ufshcd_enable_wb_if_scaling_up(hba) && !err)
1391                 ufshcd_wb_toggle(hba, scale_up);
1392
1393         mutex_unlock(&hba->wb_mutex);
1394
1395         blk_mq_unquiesce_tagset(&hba->host->tag_set);
1396         ufshcd_release(hba);
1397 }
1398
1399 /**
1400  * ufshcd_devfreq_scale - scale up/down UFS clocks and gear
1401  * @hba: per adapter instance
1402  * @freq: frequency to scale
1403  * @scale_up: True for scaling up and false for scalin down
1404  *
1405  * Return: 0 for success; -EBUSY if scaling can't happen at this time; non-zero
1406  * for any other errors.
1407  */
1408 static int ufshcd_devfreq_scale(struct ufs_hba *hba, unsigned long freq,
1409                                 bool scale_up)
1410 {
1411         int ret = 0;
1412
1413         ret = ufshcd_clock_scaling_prepare(hba, 1 * USEC_PER_SEC);
1414         if (ret)
1415                 return ret;
1416
1417         /* scale down the gear before scaling down clocks */
1418         if (!scale_up) {
1419                 ret = ufshcd_scale_gear(hba, false);
1420                 if (ret)
1421                         goto out_unprepare;
1422         }
1423
1424         ret = ufshcd_scale_clks(hba, freq, scale_up);
1425         if (ret) {
1426                 if (!scale_up)
1427                         ufshcd_scale_gear(hba, true);
1428                 goto out_unprepare;
1429         }
1430
1431         /* scale up the gear after scaling up clocks */
1432         if (scale_up) {
1433                 ret = ufshcd_scale_gear(hba, true);
1434                 if (ret) {
1435                         ufshcd_scale_clks(hba, hba->devfreq->previous_freq,
1436                                           false);
1437                         goto out_unprepare;
1438                 }
1439         }
1440
1441 out_unprepare:
1442         ufshcd_clock_scaling_unprepare(hba, ret, scale_up);
1443         return ret;
1444 }
1445
1446 static void ufshcd_clk_scaling_suspend_work(struct work_struct *work)
1447 {
1448         struct ufs_hba *hba = container_of(work, struct ufs_hba,
1449                                            clk_scaling.suspend_work);
1450         unsigned long irq_flags;
1451
1452         spin_lock_irqsave(hba->host->host_lock, irq_flags);
1453         if (hba->clk_scaling.active_reqs || hba->clk_scaling.is_suspended) {
1454                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1455                 return;
1456         }
1457         hba->clk_scaling.is_suspended = true;
1458         hba->clk_scaling.window_start_t = 0;
1459         spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1460
1461         devfreq_suspend_device(hba->devfreq);
1462 }
1463
1464 static void ufshcd_clk_scaling_resume_work(struct work_struct *work)
1465 {
1466         struct ufs_hba *hba = container_of(work, struct ufs_hba,
1467                                            clk_scaling.resume_work);
1468         unsigned long irq_flags;
1469
1470         spin_lock_irqsave(hba->host->host_lock, irq_flags);
1471         if (!hba->clk_scaling.is_suspended) {
1472                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1473                 return;
1474         }
1475         hba->clk_scaling.is_suspended = false;
1476         spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1477
1478         devfreq_resume_device(hba->devfreq);
1479 }
1480
1481 static int ufshcd_devfreq_target(struct device *dev,
1482                                 unsigned long *freq, u32 flags)
1483 {
1484         int ret = 0;
1485         struct ufs_hba *hba = dev_get_drvdata(dev);
1486         ktime_t start;
1487         bool scale_up = false, sched_clk_scaling_suspend_work = false;
1488         struct list_head *clk_list = &hba->clk_list_head;
1489         struct ufs_clk_info *clki;
1490         unsigned long irq_flags;
1491
1492         if (!ufshcd_is_clkscaling_supported(hba))
1493                 return -EINVAL;
1494
1495         if (hba->use_pm_opp) {
1496                 struct dev_pm_opp *opp;
1497
1498                 /* Get the recommended frequency from OPP framework */
1499                 opp = devfreq_recommended_opp(dev, freq, flags);
1500                 if (IS_ERR(opp))
1501                         return PTR_ERR(opp);
1502
1503                 dev_pm_opp_put(opp);
1504         } else {
1505                 /* Override with the closest supported frequency */
1506                 clki = list_first_entry(&hba->clk_list_head, struct ufs_clk_info,
1507                                         list);
1508                 *freq = (unsigned long) clk_round_rate(clki->clk, *freq);
1509         }
1510
1511         spin_lock_irqsave(hba->host->host_lock, irq_flags);
1512         if (ufshcd_eh_in_progress(hba)) {
1513                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1514                 return 0;
1515         }
1516
1517         /* Skip scaling clock when clock scaling is suspended */
1518         if (hba->clk_scaling.is_suspended) {
1519                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1520                 dev_warn(hba->dev, "clock scaling is suspended, skip");
1521                 return 0;
1522         }
1523
1524         if (!hba->clk_scaling.active_reqs)
1525                 sched_clk_scaling_suspend_work = true;
1526
1527         if (list_empty(clk_list)) {
1528                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1529                 goto out;
1530         }
1531
1532         /* Decide based on the target or rounded-off frequency and update */
1533         if (hba->use_pm_opp)
1534                 scale_up = *freq > hba->clk_scaling.target_freq;
1535         else
1536                 scale_up = *freq == clki->max_freq;
1537
1538         if (!hba->use_pm_opp && !scale_up)
1539                 *freq = clki->min_freq;
1540
1541         /* Update the frequency */
1542         if (!ufshcd_is_devfreq_scaling_required(hba, *freq, scale_up)) {
1543                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1544                 ret = 0;
1545                 goto out; /* no state change required */
1546         }
1547         spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1548
1549         start = ktime_get();
1550         ret = ufshcd_devfreq_scale(hba, *freq, scale_up);
1551         if (!ret)
1552                 hba->clk_scaling.target_freq = *freq;
1553
1554         trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
1555                 (scale_up ? "up" : "down"),
1556                 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
1557
1558 out:
1559         if (sched_clk_scaling_suspend_work &&
1560                         (!scale_up || hba->clk_scaling.suspend_on_no_request))
1561                 queue_work(hba->clk_scaling.workq,
1562                            &hba->clk_scaling.suspend_work);
1563
1564         return ret;
1565 }
1566
1567 static int ufshcd_devfreq_get_dev_status(struct device *dev,
1568                 struct devfreq_dev_status *stat)
1569 {
1570         struct ufs_hba *hba = dev_get_drvdata(dev);
1571         struct ufs_clk_scaling *scaling = &hba->clk_scaling;
1572         unsigned long flags;
1573         ktime_t curr_t;
1574
1575         if (!ufshcd_is_clkscaling_supported(hba))
1576                 return -EINVAL;
1577
1578         memset(stat, 0, sizeof(*stat));
1579
1580         spin_lock_irqsave(hba->host->host_lock, flags);
1581         curr_t = ktime_get();
1582         if (!scaling->window_start_t)
1583                 goto start_window;
1584
1585         /*
1586          * If current frequency is 0, then the ondemand governor considers
1587          * there's no initial frequency set. And it always requests to set
1588          * to max. frequency.
1589          */
1590         if (hba->use_pm_opp) {
1591                 stat->current_frequency = hba->clk_scaling.target_freq;
1592         } else {
1593                 struct list_head *clk_list = &hba->clk_list_head;
1594                 struct ufs_clk_info *clki;
1595
1596                 clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1597                 stat->current_frequency = clki->curr_freq;
1598         }
1599
1600         if (scaling->is_busy_started)
1601                 scaling->tot_busy_t += ktime_us_delta(curr_t,
1602                                 scaling->busy_start_t);
1603         stat->total_time = ktime_us_delta(curr_t, scaling->window_start_t);
1604         stat->busy_time = scaling->tot_busy_t;
1605 start_window:
1606         scaling->window_start_t = curr_t;
1607         scaling->tot_busy_t = 0;
1608
1609         if (scaling->active_reqs) {
1610                 scaling->busy_start_t = curr_t;
1611                 scaling->is_busy_started = true;
1612         } else {
1613                 scaling->busy_start_t = 0;
1614                 scaling->is_busy_started = false;
1615         }
1616         spin_unlock_irqrestore(hba->host->host_lock, flags);
1617         return 0;
1618 }
1619
1620 static int ufshcd_devfreq_init(struct ufs_hba *hba)
1621 {
1622         struct list_head *clk_list = &hba->clk_list_head;
1623         struct ufs_clk_info *clki;
1624         struct devfreq *devfreq;
1625         int ret;
1626
1627         /* Skip devfreq if we don't have any clocks in the list */
1628         if (list_empty(clk_list))
1629                 return 0;
1630
1631         if (!hba->use_pm_opp) {
1632                 clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1633                 dev_pm_opp_add(hba->dev, clki->min_freq, 0);
1634                 dev_pm_opp_add(hba->dev, clki->max_freq, 0);
1635         }
1636
1637         ufshcd_vops_config_scaling_param(hba, &hba->vps->devfreq_profile,
1638                                          &hba->vps->ondemand_data);
1639         devfreq = devfreq_add_device(hba->dev,
1640                         &hba->vps->devfreq_profile,
1641                         DEVFREQ_GOV_SIMPLE_ONDEMAND,
1642                         &hba->vps->ondemand_data);
1643         if (IS_ERR(devfreq)) {
1644                 ret = PTR_ERR(devfreq);
1645                 dev_err(hba->dev, "Unable to register with devfreq %d\n", ret);
1646
1647                 if (!hba->use_pm_opp) {
1648                         dev_pm_opp_remove(hba->dev, clki->min_freq);
1649                         dev_pm_opp_remove(hba->dev, clki->max_freq);
1650                 }
1651                 return ret;
1652         }
1653
1654         hba->devfreq = devfreq;
1655
1656         return 0;
1657 }
1658
1659 static void ufshcd_devfreq_remove(struct ufs_hba *hba)
1660 {
1661         struct list_head *clk_list = &hba->clk_list_head;
1662
1663         if (!hba->devfreq)
1664                 return;
1665
1666         devfreq_remove_device(hba->devfreq);
1667         hba->devfreq = NULL;
1668
1669         if (!hba->use_pm_opp) {
1670                 struct ufs_clk_info *clki;
1671
1672                 clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1673                 dev_pm_opp_remove(hba->dev, clki->min_freq);
1674                 dev_pm_opp_remove(hba->dev, clki->max_freq);
1675         }
1676 }
1677
1678 static void ufshcd_suspend_clkscaling(struct ufs_hba *hba)
1679 {
1680         unsigned long flags;
1681         bool suspend = false;
1682
1683         cancel_work_sync(&hba->clk_scaling.suspend_work);
1684         cancel_work_sync(&hba->clk_scaling.resume_work);
1685
1686         spin_lock_irqsave(hba->host->host_lock, flags);
1687         if (!hba->clk_scaling.is_suspended) {
1688                 suspend = true;
1689                 hba->clk_scaling.is_suspended = true;
1690                 hba->clk_scaling.window_start_t = 0;
1691         }
1692         spin_unlock_irqrestore(hba->host->host_lock, flags);
1693
1694         if (suspend)
1695                 devfreq_suspend_device(hba->devfreq);
1696 }
1697
1698 static void ufshcd_resume_clkscaling(struct ufs_hba *hba)
1699 {
1700         unsigned long flags;
1701         bool resume = false;
1702
1703         spin_lock_irqsave(hba->host->host_lock, flags);
1704         if (hba->clk_scaling.is_suspended) {
1705                 resume = true;
1706                 hba->clk_scaling.is_suspended = false;
1707         }
1708         spin_unlock_irqrestore(hba->host->host_lock, flags);
1709
1710         if (resume)
1711                 devfreq_resume_device(hba->devfreq);
1712 }
1713
1714 static ssize_t ufshcd_clkscale_enable_show(struct device *dev,
1715                 struct device_attribute *attr, char *buf)
1716 {
1717         struct ufs_hba *hba = dev_get_drvdata(dev);
1718
1719         return sysfs_emit(buf, "%d\n", hba->clk_scaling.is_enabled);
1720 }
1721
1722 static ssize_t ufshcd_clkscale_enable_store(struct device *dev,
1723                 struct device_attribute *attr, const char *buf, size_t count)
1724 {
1725         struct ufs_hba *hba = dev_get_drvdata(dev);
1726         u32 value;
1727         int err = 0;
1728
1729         if (kstrtou32(buf, 0, &value))
1730                 return -EINVAL;
1731
1732         down(&hba->host_sem);
1733         if (!ufshcd_is_user_access_allowed(hba)) {
1734                 err = -EBUSY;
1735                 goto out;
1736         }
1737
1738         value = !!value;
1739         if (value == hba->clk_scaling.is_enabled)
1740                 goto out;
1741
1742         ufshcd_rpm_get_sync(hba);
1743         ufshcd_hold(hba);
1744
1745         hba->clk_scaling.is_enabled = value;
1746
1747         if (value) {
1748                 ufshcd_resume_clkscaling(hba);
1749         } else {
1750                 ufshcd_suspend_clkscaling(hba);
1751                 err = ufshcd_devfreq_scale(hba, ULONG_MAX, true);
1752                 if (err)
1753                         dev_err(hba->dev, "%s: failed to scale clocks up %d\n",
1754                                         __func__, err);
1755         }
1756
1757         ufshcd_release(hba);
1758         ufshcd_rpm_put_sync(hba);
1759 out:
1760         up(&hba->host_sem);
1761         return err ? err : count;
1762 }
1763
1764 static void ufshcd_init_clk_scaling_sysfs(struct ufs_hba *hba)
1765 {
1766         hba->clk_scaling.enable_attr.show = ufshcd_clkscale_enable_show;
1767         hba->clk_scaling.enable_attr.store = ufshcd_clkscale_enable_store;
1768         sysfs_attr_init(&hba->clk_scaling.enable_attr.attr);
1769         hba->clk_scaling.enable_attr.attr.name = "clkscale_enable";
1770         hba->clk_scaling.enable_attr.attr.mode = 0644;
1771         if (device_create_file(hba->dev, &hba->clk_scaling.enable_attr))
1772                 dev_err(hba->dev, "Failed to create sysfs for clkscale_enable\n");
1773 }
1774
1775 static void ufshcd_remove_clk_scaling_sysfs(struct ufs_hba *hba)
1776 {
1777         if (hba->clk_scaling.enable_attr.attr.name)
1778                 device_remove_file(hba->dev, &hba->clk_scaling.enable_attr);
1779 }
1780
1781 static void ufshcd_init_clk_scaling(struct ufs_hba *hba)
1782 {
1783         if (!ufshcd_is_clkscaling_supported(hba))
1784                 return;
1785
1786         if (!hba->clk_scaling.min_gear)
1787                 hba->clk_scaling.min_gear = UFS_HS_G1;
1788
1789         INIT_WORK(&hba->clk_scaling.suspend_work,
1790                   ufshcd_clk_scaling_suspend_work);
1791         INIT_WORK(&hba->clk_scaling.resume_work,
1792                   ufshcd_clk_scaling_resume_work);
1793
1794         hba->clk_scaling.workq = alloc_ordered_workqueue(
1795                 "ufs_clkscaling_%d", WQ_MEM_RECLAIM, hba->host->host_no);
1796
1797         hba->clk_scaling.is_initialized = true;
1798 }
1799
1800 static void ufshcd_exit_clk_scaling(struct ufs_hba *hba)
1801 {
1802         if (!hba->clk_scaling.is_initialized)
1803                 return;
1804
1805         ufshcd_remove_clk_scaling_sysfs(hba);
1806         destroy_workqueue(hba->clk_scaling.workq);
1807         ufshcd_devfreq_remove(hba);
1808         hba->clk_scaling.is_initialized = false;
1809 }
1810
1811 static void ufshcd_ungate_work(struct work_struct *work)
1812 {
1813         int ret;
1814         unsigned long flags;
1815         struct ufs_hba *hba = container_of(work, struct ufs_hba,
1816                         clk_gating.ungate_work);
1817
1818         cancel_delayed_work_sync(&hba->clk_gating.gate_work);
1819
1820         spin_lock_irqsave(hba->host->host_lock, flags);
1821         if (hba->clk_gating.state == CLKS_ON) {
1822                 spin_unlock_irqrestore(hba->host->host_lock, flags);
1823                 return;
1824         }
1825
1826         spin_unlock_irqrestore(hba->host->host_lock, flags);
1827         ufshcd_hba_vreg_set_hpm(hba);
1828         ufshcd_setup_clocks(hba, true);
1829
1830         ufshcd_enable_irq(hba);
1831
1832         /* Exit from hibern8 */
1833         if (ufshcd_can_hibern8_during_gating(hba)) {
1834                 /* Prevent gating in this path */
1835                 hba->clk_gating.is_suspended = true;
1836                 if (ufshcd_is_link_hibern8(hba)) {
1837                         ret = ufshcd_uic_hibern8_exit(hba);
1838                         if (ret)
1839                                 dev_err(hba->dev, "%s: hibern8 exit failed %d\n",
1840                                         __func__, ret);
1841                         else
1842                                 ufshcd_set_link_active(hba);
1843                 }
1844                 hba->clk_gating.is_suspended = false;
1845         }
1846 }
1847
1848 /**
1849  * ufshcd_hold - Enable clocks that were gated earlier due to ufshcd_release.
1850  * Also, exit from hibern8 mode and set the link as active.
1851  * @hba: per adapter instance
1852  */
1853 void ufshcd_hold(struct ufs_hba *hba)
1854 {
1855         bool flush_result;
1856         unsigned long flags;
1857
1858         if (!ufshcd_is_clkgating_allowed(hba) ||
1859             !hba->clk_gating.is_initialized)
1860                 return;
1861         spin_lock_irqsave(hba->host->host_lock, flags);
1862         hba->clk_gating.active_reqs++;
1863
1864 start:
1865         switch (hba->clk_gating.state) {
1866         case CLKS_ON:
1867                 /*
1868                  * Wait for the ungate work to complete if in progress.
1869                  * Though the clocks may be in ON state, the link could
1870                  * still be in hibner8 state if hibern8 is allowed
1871                  * during clock gating.
1872                  * Make sure we exit hibern8 state also in addition to
1873                  * clocks being ON.
1874                  */
1875                 if (ufshcd_can_hibern8_during_gating(hba) &&
1876                     ufshcd_is_link_hibern8(hba)) {
1877                         spin_unlock_irqrestore(hba->host->host_lock, flags);
1878                         flush_result = flush_work(&hba->clk_gating.ungate_work);
1879                         if (hba->clk_gating.is_suspended && !flush_result)
1880                                 return;
1881                         spin_lock_irqsave(hba->host->host_lock, flags);
1882                         goto start;
1883                 }
1884                 break;
1885         case REQ_CLKS_OFF:
1886                 if (cancel_delayed_work(&hba->clk_gating.gate_work)) {
1887                         hba->clk_gating.state = CLKS_ON;
1888                         trace_ufshcd_clk_gating(dev_name(hba->dev),
1889                                                 hba->clk_gating.state);
1890                         break;
1891                 }
1892                 /*
1893                  * If we are here, it means gating work is either done or
1894                  * currently running. Hence, fall through to cancel gating
1895                  * work and to enable clocks.
1896                  */
1897                 fallthrough;
1898         case CLKS_OFF:
1899                 hba->clk_gating.state = REQ_CLKS_ON;
1900                 trace_ufshcd_clk_gating(dev_name(hba->dev),
1901                                         hba->clk_gating.state);
1902                 queue_work(hba->clk_gating.clk_gating_workq,
1903                            &hba->clk_gating.ungate_work);
1904                 /*
1905                  * fall through to check if we should wait for this
1906                  * work to be done or not.
1907                  */
1908                 fallthrough;
1909         case REQ_CLKS_ON:
1910                 spin_unlock_irqrestore(hba->host->host_lock, flags);
1911                 flush_work(&hba->clk_gating.ungate_work);
1912                 /* Make sure state is CLKS_ON before returning */
1913                 spin_lock_irqsave(hba->host->host_lock, flags);
1914                 goto start;
1915         default:
1916                 dev_err(hba->dev, "%s: clk gating is in invalid state %d\n",
1917                                 __func__, hba->clk_gating.state);
1918                 break;
1919         }
1920         spin_unlock_irqrestore(hba->host->host_lock, flags);
1921 }
1922 EXPORT_SYMBOL_GPL(ufshcd_hold);
1923
1924 static void ufshcd_gate_work(struct work_struct *work)
1925 {
1926         struct ufs_hba *hba = container_of(work, struct ufs_hba,
1927                         clk_gating.gate_work.work);
1928         unsigned long flags;
1929         int ret;
1930
1931         spin_lock_irqsave(hba->host->host_lock, flags);
1932         /*
1933          * In case you are here to cancel this work the gating state
1934          * would be marked as REQ_CLKS_ON. In this case save time by
1935          * skipping the gating work and exit after changing the clock
1936          * state to CLKS_ON.
1937          */
1938         if (hba->clk_gating.is_suspended ||
1939                 (hba->clk_gating.state != REQ_CLKS_OFF)) {
1940                 hba->clk_gating.state = CLKS_ON;
1941                 trace_ufshcd_clk_gating(dev_name(hba->dev),
1942                                         hba->clk_gating.state);
1943                 goto rel_lock;
1944         }
1945
1946         if (ufshcd_is_ufs_dev_busy(hba) || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL)
1947                 goto rel_lock;
1948
1949         spin_unlock_irqrestore(hba->host->host_lock, flags);
1950
1951         /* put the link into hibern8 mode before turning off clocks */
1952         if (ufshcd_can_hibern8_during_gating(hba)) {
1953                 ret = ufshcd_uic_hibern8_enter(hba);
1954                 if (ret) {
1955                         hba->clk_gating.state = CLKS_ON;
1956                         dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
1957                                         __func__, ret);
1958                         trace_ufshcd_clk_gating(dev_name(hba->dev),
1959                                                 hba->clk_gating.state);
1960                         goto out;
1961                 }
1962                 ufshcd_set_link_hibern8(hba);
1963         }
1964
1965         ufshcd_disable_irq(hba);
1966
1967         ufshcd_setup_clocks(hba, false);
1968
1969         /* Put the host controller in low power mode if possible */
1970         ufshcd_hba_vreg_set_lpm(hba);
1971         /*
1972          * In case you are here to cancel this work the gating state
1973          * would be marked as REQ_CLKS_ON. In this case keep the state
1974          * as REQ_CLKS_ON which would anyway imply that clocks are off
1975          * and a request to turn them on is pending. By doing this way,
1976          * we keep the state machine in tact and this would ultimately
1977          * prevent from doing cancel work multiple times when there are
1978          * new requests arriving before the current cancel work is done.
1979          */
1980         spin_lock_irqsave(hba->host->host_lock, flags);
1981         if (hba->clk_gating.state == REQ_CLKS_OFF) {
1982                 hba->clk_gating.state = CLKS_OFF;
1983                 trace_ufshcd_clk_gating(dev_name(hba->dev),
1984                                         hba->clk_gating.state);
1985         }
1986 rel_lock:
1987         spin_unlock_irqrestore(hba->host->host_lock, flags);
1988 out:
1989         return;
1990 }
1991
1992 /* host lock must be held before calling this variant */
1993 static void __ufshcd_release(struct ufs_hba *hba)
1994 {
1995         if (!ufshcd_is_clkgating_allowed(hba))
1996                 return;
1997
1998         hba->clk_gating.active_reqs--;
1999
2000         if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended ||
2001             hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL ||
2002             hba->outstanding_tasks || !hba->clk_gating.is_initialized ||
2003             hba->active_uic_cmd || hba->uic_async_done ||
2004             hba->clk_gating.state == CLKS_OFF)
2005                 return;
2006
2007         hba->clk_gating.state = REQ_CLKS_OFF;
2008         trace_ufshcd_clk_gating(dev_name(hba->dev), hba->clk_gating.state);
2009         queue_delayed_work(hba->clk_gating.clk_gating_workq,
2010                            &hba->clk_gating.gate_work,
2011                            msecs_to_jiffies(hba->clk_gating.delay_ms));
2012 }
2013
2014 void ufshcd_release(struct ufs_hba *hba)
2015 {
2016         unsigned long flags;
2017
2018         spin_lock_irqsave(hba->host->host_lock, flags);
2019         __ufshcd_release(hba);
2020         spin_unlock_irqrestore(hba->host->host_lock, flags);
2021 }
2022 EXPORT_SYMBOL_GPL(ufshcd_release);
2023
2024 static ssize_t ufshcd_clkgate_delay_show(struct device *dev,
2025                 struct device_attribute *attr, char *buf)
2026 {
2027         struct ufs_hba *hba = dev_get_drvdata(dev);
2028
2029         return sysfs_emit(buf, "%lu\n", hba->clk_gating.delay_ms);
2030 }
2031
2032 void ufshcd_clkgate_delay_set(struct device *dev, unsigned long value)
2033 {
2034         struct ufs_hba *hba = dev_get_drvdata(dev);
2035         unsigned long flags;
2036
2037         spin_lock_irqsave(hba->host->host_lock, flags);
2038         hba->clk_gating.delay_ms = value;
2039         spin_unlock_irqrestore(hba->host->host_lock, flags);
2040 }
2041 EXPORT_SYMBOL_GPL(ufshcd_clkgate_delay_set);
2042
2043 static ssize_t ufshcd_clkgate_delay_store(struct device *dev,
2044                 struct device_attribute *attr, const char *buf, size_t count)
2045 {
2046         unsigned long value;
2047
2048         if (kstrtoul(buf, 0, &value))
2049                 return -EINVAL;
2050
2051         ufshcd_clkgate_delay_set(dev, value);
2052         return count;
2053 }
2054
2055 static ssize_t ufshcd_clkgate_enable_show(struct device *dev,
2056                 struct device_attribute *attr, char *buf)
2057 {
2058         struct ufs_hba *hba = dev_get_drvdata(dev);
2059
2060         return sysfs_emit(buf, "%d\n", hba->clk_gating.is_enabled);
2061 }
2062
2063 static ssize_t ufshcd_clkgate_enable_store(struct device *dev,
2064                 struct device_attribute *attr, const char *buf, size_t count)
2065 {
2066         struct ufs_hba *hba = dev_get_drvdata(dev);
2067         unsigned long flags;
2068         u32 value;
2069
2070         if (kstrtou32(buf, 0, &value))
2071                 return -EINVAL;
2072
2073         value = !!value;
2074
2075         spin_lock_irqsave(hba->host->host_lock, flags);
2076         if (value == hba->clk_gating.is_enabled)
2077                 goto out;
2078
2079         if (value)
2080                 __ufshcd_release(hba);
2081         else
2082                 hba->clk_gating.active_reqs++;
2083
2084         hba->clk_gating.is_enabled = value;
2085 out:
2086         spin_unlock_irqrestore(hba->host->host_lock, flags);
2087         return count;
2088 }
2089
2090 static void ufshcd_init_clk_gating_sysfs(struct ufs_hba *hba)
2091 {
2092         hba->clk_gating.delay_attr.show = ufshcd_clkgate_delay_show;
2093         hba->clk_gating.delay_attr.store = ufshcd_clkgate_delay_store;
2094         sysfs_attr_init(&hba->clk_gating.delay_attr.attr);
2095         hba->clk_gating.delay_attr.attr.name = "clkgate_delay_ms";
2096         hba->clk_gating.delay_attr.attr.mode = 0644;
2097         if (device_create_file(hba->dev, &hba->clk_gating.delay_attr))
2098                 dev_err(hba->dev, "Failed to create sysfs for clkgate_delay\n");
2099
2100         hba->clk_gating.enable_attr.show = ufshcd_clkgate_enable_show;
2101         hba->clk_gating.enable_attr.store = ufshcd_clkgate_enable_store;
2102         sysfs_attr_init(&hba->clk_gating.enable_attr.attr);
2103         hba->clk_gating.enable_attr.attr.name = "clkgate_enable";
2104         hba->clk_gating.enable_attr.attr.mode = 0644;
2105         if (device_create_file(hba->dev, &hba->clk_gating.enable_attr))
2106                 dev_err(hba->dev, "Failed to create sysfs for clkgate_enable\n");
2107 }
2108
2109 static void ufshcd_remove_clk_gating_sysfs(struct ufs_hba *hba)
2110 {
2111         if (hba->clk_gating.delay_attr.attr.name)
2112                 device_remove_file(hba->dev, &hba->clk_gating.delay_attr);
2113         if (hba->clk_gating.enable_attr.attr.name)
2114                 device_remove_file(hba->dev, &hba->clk_gating.enable_attr);
2115 }
2116
2117 static void ufshcd_init_clk_gating(struct ufs_hba *hba)
2118 {
2119         if (!ufshcd_is_clkgating_allowed(hba))
2120                 return;
2121
2122         hba->clk_gating.state = CLKS_ON;
2123
2124         hba->clk_gating.delay_ms = 150;
2125         INIT_DELAYED_WORK(&hba->clk_gating.gate_work, ufshcd_gate_work);
2126         INIT_WORK(&hba->clk_gating.ungate_work, ufshcd_ungate_work);
2127
2128         hba->clk_gating.clk_gating_workq = alloc_ordered_workqueue(
2129                 "ufs_clk_gating_%d", WQ_MEM_RECLAIM | WQ_HIGHPRI,
2130                 hba->host->host_no);
2131
2132         ufshcd_init_clk_gating_sysfs(hba);
2133
2134         hba->clk_gating.is_enabled = true;
2135         hba->clk_gating.is_initialized = true;
2136 }
2137
2138 static void ufshcd_exit_clk_gating(struct ufs_hba *hba)
2139 {
2140         if (!hba->clk_gating.is_initialized)
2141                 return;
2142
2143         ufshcd_remove_clk_gating_sysfs(hba);
2144
2145         /* Ungate the clock if necessary. */
2146         ufshcd_hold(hba);
2147         hba->clk_gating.is_initialized = false;
2148         ufshcd_release(hba);
2149
2150         destroy_workqueue(hba->clk_gating.clk_gating_workq);
2151 }
2152
2153 static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba)
2154 {
2155         bool queue_resume_work = false;
2156         ktime_t curr_t = ktime_get();
2157         unsigned long flags;
2158
2159         if (!ufshcd_is_clkscaling_supported(hba))
2160                 return;
2161
2162         spin_lock_irqsave(hba->host->host_lock, flags);
2163         if (!hba->clk_scaling.active_reqs++)
2164                 queue_resume_work = true;
2165
2166         if (!hba->clk_scaling.is_enabled || hba->pm_op_in_progress) {
2167                 spin_unlock_irqrestore(hba->host->host_lock, flags);
2168                 return;
2169         }
2170
2171         if (queue_resume_work)
2172                 queue_work(hba->clk_scaling.workq,
2173                            &hba->clk_scaling.resume_work);
2174
2175         if (!hba->clk_scaling.window_start_t) {
2176                 hba->clk_scaling.window_start_t = curr_t;
2177                 hba->clk_scaling.tot_busy_t = 0;
2178                 hba->clk_scaling.is_busy_started = false;
2179         }
2180
2181         if (!hba->clk_scaling.is_busy_started) {
2182                 hba->clk_scaling.busy_start_t = curr_t;
2183                 hba->clk_scaling.is_busy_started = true;
2184         }
2185         spin_unlock_irqrestore(hba->host->host_lock, flags);
2186 }
2187
2188 static void ufshcd_clk_scaling_update_busy(struct ufs_hba *hba)
2189 {
2190         struct ufs_clk_scaling *scaling = &hba->clk_scaling;
2191         unsigned long flags;
2192
2193         if (!ufshcd_is_clkscaling_supported(hba))
2194                 return;
2195
2196         spin_lock_irqsave(hba->host->host_lock, flags);
2197         hba->clk_scaling.active_reqs--;
2198         if (!scaling->active_reqs && scaling->is_busy_started) {
2199                 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(),
2200                                         scaling->busy_start_t));
2201                 scaling->busy_start_t = 0;
2202                 scaling->is_busy_started = false;
2203         }
2204         spin_unlock_irqrestore(hba->host->host_lock, flags);
2205 }
2206
2207 static inline int ufshcd_monitor_opcode2dir(u8 opcode)
2208 {
2209         if (opcode == READ_6 || opcode == READ_10 || opcode == READ_16)
2210                 return READ;
2211         else if (opcode == WRITE_6 || opcode == WRITE_10 || opcode == WRITE_16)
2212                 return WRITE;
2213         else
2214                 return -EINVAL;
2215 }
2216
2217 static inline bool ufshcd_should_inform_monitor(struct ufs_hba *hba,
2218                                                 struct ufshcd_lrb *lrbp)
2219 {
2220         const struct ufs_hba_monitor *m = &hba->monitor;
2221
2222         return (m->enabled && lrbp && lrbp->cmd &&
2223                 (!m->chunk_size || m->chunk_size == lrbp->cmd->sdb.length) &&
2224                 ktime_before(hba->monitor.enabled_ts, lrbp->issue_time_stamp));
2225 }
2226
2227 static void ufshcd_start_monitor(struct ufs_hba *hba,
2228                                  const struct ufshcd_lrb *lrbp)
2229 {
2230         int dir = ufshcd_monitor_opcode2dir(*lrbp->cmd->cmnd);
2231         unsigned long flags;
2232
2233         spin_lock_irqsave(hba->host->host_lock, flags);
2234         if (dir >= 0 && hba->monitor.nr_queued[dir]++ == 0)
2235                 hba->monitor.busy_start_ts[dir] = ktime_get();
2236         spin_unlock_irqrestore(hba->host->host_lock, flags);
2237 }
2238
2239 static void ufshcd_update_monitor(struct ufs_hba *hba, const struct ufshcd_lrb *lrbp)
2240 {
2241         int dir = ufshcd_monitor_opcode2dir(*lrbp->cmd->cmnd);
2242         unsigned long flags;
2243
2244         spin_lock_irqsave(hba->host->host_lock, flags);
2245         if (dir >= 0 && hba->monitor.nr_queued[dir] > 0) {
2246                 const struct request *req = scsi_cmd_to_rq(lrbp->cmd);
2247                 struct ufs_hba_monitor *m = &hba->monitor;
2248                 ktime_t now, inc, lat;
2249
2250                 now = lrbp->compl_time_stamp;
2251                 inc = ktime_sub(now, m->busy_start_ts[dir]);
2252                 m->total_busy[dir] = ktime_add(m->total_busy[dir], inc);
2253                 m->nr_sec_rw[dir] += blk_rq_sectors(req);
2254
2255                 /* Update latencies */
2256                 m->nr_req[dir]++;
2257                 lat = ktime_sub(now, lrbp->issue_time_stamp);
2258                 m->lat_sum[dir] += lat;
2259                 if (m->lat_max[dir] < lat || !m->lat_max[dir])
2260                         m->lat_max[dir] = lat;
2261                 if (m->lat_min[dir] > lat || !m->lat_min[dir])
2262                         m->lat_min[dir] = lat;
2263
2264                 m->nr_queued[dir]--;
2265                 /* Push forward the busy start of monitor */
2266                 m->busy_start_ts[dir] = now;
2267         }
2268         spin_unlock_irqrestore(hba->host->host_lock, flags);
2269 }
2270
2271 /**
2272  * ufshcd_send_command - Send SCSI or device management commands
2273  * @hba: per adapter instance
2274  * @task_tag: Task tag of the command
2275  * @hwq: pointer to hardware queue instance
2276  */
2277 static inline
2278 void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag,
2279                          struct ufs_hw_queue *hwq)
2280 {
2281         struct ufshcd_lrb *lrbp = &hba->lrb[task_tag];
2282         unsigned long flags;
2283
2284         lrbp->issue_time_stamp = ktime_get();
2285         lrbp->issue_time_stamp_local_clock = local_clock();
2286         lrbp->compl_time_stamp = ktime_set(0, 0);
2287         lrbp->compl_time_stamp_local_clock = 0;
2288         ufshcd_add_command_trace(hba, task_tag, UFS_CMD_SEND);
2289         if (lrbp->cmd)
2290                 ufshcd_clk_scaling_start_busy(hba);
2291         if (unlikely(ufshcd_should_inform_monitor(hba, lrbp)))
2292                 ufshcd_start_monitor(hba, lrbp);
2293
2294         if (hba->mcq_enabled) {
2295                 int utrd_size = sizeof(struct utp_transfer_req_desc);
2296                 struct utp_transfer_req_desc *src = lrbp->utr_descriptor_ptr;
2297                 struct utp_transfer_req_desc *dest;
2298
2299                 spin_lock(&hwq->sq_lock);
2300                 dest = hwq->sqe_base_addr + hwq->sq_tail_slot;
2301                 memcpy(dest, src, utrd_size);
2302                 ufshcd_inc_sq_tail(hwq);
2303                 spin_unlock(&hwq->sq_lock);
2304         } else {
2305                 spin_lock_irqsave(&hba->outstanding_lock, flags);
2306                 if (hba->vops && hba->vops->setup_xfer_req)
2307                         hba->vops->setup_xfer_req(hba, lrbp->task_tag,
2308                                                   !!lrbp->cmd);
2309                 __set_bit(lrbp->task_tag, &hba->outstanding_reqs);
2310                 ufshcd_writel(hba, 1 << lrbp->task_tag,
2311                               REG_UTP_TRANSFER_REQ_DOOR_BELL);
2312                 spin_unlock_irqrestore(&hba->outstanding_lock, flags);
2313         }
2314 }
2315
2316 /**
2317  * ufshcd_copy_sense_data - Copy sense data in case of check condition
2318  * @lrbp: pointer to local reference block
2319  */
2320 static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp)
2321 {
2322         u8 *const sense_buffer = lrbp->cmd->sense_buffer;
2323         u16 resp_len;
2324         int len;
2325
2326         resp_len = be16_to_cpu(lrbp->ucd_rsp_ptr->header.data_segment_length);
2327         if (sense_buffer && resp_len) {
2328                 int len_to_copy;
2329
2330                 len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len);
2331                 len_to_copy = min_t(int, UFS_SENSE_SIZE, len);
2332
2333                 memcpy(sense_buffer, lrbp->ucd_rsp_ptr->sr.sense_data,
2334                        len_to_copy);
2335         }
2336 }
2337
2338 /**
2339  * ufshcd_copy_query_response() - Copy the Query Response and the data
2340  * descriptor
2341  * @hba: per adapter instance
2342  * @lrbp: pointer to local reference block
2343  *
2344  * Return: 0 upon success; < 0 upon failure.
2345  */
2346 static
2347 int ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2348 {
2349         struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
2350
2351         memcpy(&query_res->upiu_res, &lrbp->ucd_rsp_ptr->qr, QUERY_OSF_SIZE);
2352
2353         /* Get the descriptor */
2354         if (hba->dev_cmd.query.descriptor &&
2355             lrbp->ucd_rsp_ptr->qr.opcode == UPIU_QUERY_OPCODE_READ_DESC) {
2356                 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr +
2357                                 GENERAL_UPIU_REQUEST_SIZE;
2358                 u16 resp_len;
2359                 u16 buf_len;
2360
2361                 /* data segment length */
2362                 resp_len = be16_to_cpu(lrbp->ucd_rsp_ptr->header
2363                                        .data_segment_length);
2364                 buf_len = be16_to_cpu(
2365                                 hba->dev_cmd.query.request.upiu_req.length);
2366                 if (likely(buf_len >= resp_len)) {
2367                         memcpy(hba->dev_cmd.query.descriptor, descp, resp_len);
2368                 } else {
2369                         dev_warn(hba->dev,
2370                                  "%s: rsp size %d is bigger than buffer size %d",
2371                                  __func__, resp_len, buf_len);
2372                         return -EINVAL;
2373                 }
2374         }
2375
2376         return 0;
2377 }
2378
2379 /**
2380  * ufshcd_hba_capabilities - Read controller capabilities
2381  * @hba: per adapter instance
2382  *
2383  * Return: 0 on success, negative on error.
2384  */
2385 static inline int ufshcd_hba_capabilities(struct ufs_hba *hba)
2386 {
2387         int err;
2388
2389         hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES);
2390
2391         /* nutrs and nutmrs are 0 based values */
2392         hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS_SDB) + 1;
2393         hba->nutmrs =
2394         ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1;
2395         hba->reserved_slot = hba->nutrs - 1;
2396
2397         hba->nortt = FIELD_GET(MASK_NUMBER_OUTSTANDING_RTT, hba->capabilities) + 1;
2398
2399         /* Read crypto capabilities */
2400         err = ufshcd_hba_init_crypto_capabilities(hba);
2401         if (err) {
2402                 dev_err(hba->dev, "crypto setup failed\n");
2403                 return err;
2404         }
2405
2406         /*
2407          * The UFSHCI 3.0 specification does not define MCQ_SUPPORT and
2408          * LSDB_SUPPORT, but [31:29] as reserved bits with reset value 0s, which
2409          * means we can simply read values regardless of version.
2410          */
2411         hba->mcq_sup = FIELD_GET(MASK_MCQ_SUPPORT, hba->capabilities);
2412         /*
2413          * 0h: legacy single doorbell support is available
2414          * 1h: indicate that legacy single doorbell support has been removed
2415          */
2416         if (!(hba->quirks & UFSHCD_QUIRK_BROKEN_LSDBS_CAP))
2417                 hba->lsdb_sup = !FIELD_GET(MASK_LSDB_SUPPORT, hba->capabilities);
2418         else
2419                 hba->lsdb_sup = true;
2420
2421         if (!hba->mcq_sup)
2422                 return 0;
2423
2424         hba->mcq_capabilities = ufshcd_readl(hba, REG_MCQCAP);
2425         hba->ext_iid_sup = FIELD_GET(MASK_EXT_IID_SUPPORT,
2426                                      hba->mcq_capabilities);
2427
2428         return 0;
2429 }
2430
2431 /**
2432  * ufshcd_ready_for_uic_cmd - Check if controller is ready
2433  *                            to accept UIC commands
2434  * @hba: per adapter instance
2435  *
2436  * Return: true on success, else false.
2437  */
2438 static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba)
2439 {
2440         u32 val;
2441         int ret = read_poll_timeout(ufshcd_readl, val, val & UIC_COMMAND_READY,
2442                                     500, uic_cmd_timeout * 1000, false, hba,
2443                                     REG_CONTROLLER_STATUS);
2444         return ret == 0;
2445 }
2446
2447 /**
2448  * ufshcd_get_upmcrs - Get the power mode change request status
2449  * @hba: Pointer to adapter instance
2450  *
2451  * This function gets the UPMCRS field of HCS register
2452  *
2453  * Return: value of UPMCRS field.
2454  */
2455 static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba)
2456 {
2457         return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) >> 8) & 0x7;
2458 }
2459
2460 /**
2461  * ufshcd_dispatch_uic_cmd - Dispatch an UIC command to the Unipro layer
2462  * @hba: per adapter instance
2463  * @uic_cmd: UIC command
2464  */
2465 static inline void
2466 ufshcd_dispatch_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2467 {
2468         lockdep_assert_held(&hba->uic_cmd_mutex);
2469
2470         WARN_ON(hba->active_uic_cmd);
2471
2472         hba->active_uic_cmd = uic_cmd;
2473
2474         /* Write Args */
2475         ufshcd_writel(hba, uic_cmd->argument1, REG_UIC_COMMAND_ARG_1);
2476         ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2);
2477         ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3);
2478
2479         ufshcd_add_uic_command_trace(hba, uic_cmd, UFS_CMD_SEND);
2480
2481         /* Write UIC Cmd */
2482         ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK,
2483                       REG_UIC_COMMAND);
2484 }
2485
2486 /**
2487  * ufshcd_wait_for_uic_cmd - Wait for completion of an UIC command
2488  * @hba: per adapter instance
2489  * @uic_cmd: UIC command
2490  *
2491  * Return: 0 only if success.
2492  */
2493 static int
2494 ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2495 {
2496         int ret;
2497         unsigned long flags;
2498
2499         lockdep_assert_held(&hba->uic_cmd_mutex);
2500
2501         if (wait_for_completion_timeout(&uic_cmd->done,
2502                                         msecs_to_jiffies(uic_cmd_timeout))) {
2503                 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
2504         } else {
2505                 ret = -ETIMEDOUT;
2506                 dev_err(hba->dev,
2507                         "uic cmd 0x%x with arg3 0x%x completion timeout\n",
2508                         uic_cmd->command, uic_cmd->argument3);
2509
2510                 if (!uic_cmd->cmd_active) {
2511                         dev_err(hba->dev, "%s: UIC cmd has been completed, return the result\n",
2512                                 __func__);
2513                         ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
2514                 }
2515         }
2516
2517         spin_lock_irqsave(hba->host->host_lock, flags);
2518         hba->active_uic_cmd = NULL;
2519         spin_unlock_irqrestore(hba->host->host_lock, flags);
2520
2521         return ret;
2522 }
2523
2524 /**
2525  * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2526  * @hba: per adapter instance
2527  * @uic_cmd: UIC command
2528  *
2529  * Return: 0 only if success.
2530  */
2531 static int
2532 __ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2533 {
2534         lockdep_assert_held(&hba->uic_cmd_mutex);
2535
2536         if (!ufshcd_ready_for_uic_cmd(hba)) {
2537                 dev_err(hba->dev,
2538                         "Controller not ready to accept UIC commands\n");
2539                 return -EIO;
2540         }
2541
2542         init_completion(&uic_cmd->done);
2543
2544         uic_cmd->cmd_active = 1;
2545         ufshcd_dispatch_uic_cmd(hba, uic_cmd);
2546
2547         return 0;
2548 }
2549
2550 /**
2551  * ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2552  * @hba: per adapter instance
2553  * @uic_cmd: UIC command
2554  *
2555  * Return: 0 only if success.
2556  */
2557 int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2558 {
2559         int ret;
2560
2561         if (hba->quirks & UFSHCD_QUIRK_BROKEN_UIC_CMD)
2562                 return 0;
2563
2564         ufshcd_hold(hba);
2565         mutex_lock(&hba->uic_cmd_mutex);
2566         ufshcd_add_delay_before_dme_cmd(hba);
2567
2568         ret = __ufshcd_send_uic_cmd(hba, uic_cmd);
2569         if (!ret)
2570                 ret = ufshcd_wait_for_uic_cmd(hba, uic_cmd);
2571
2572         mutex_unlock(&hba->uic_cmd_mutex);
2573
2574         ufshcd_release(hba);
2575         return ret;
2576 }
2577
2578 /**
2579  * ufshcd_sgl_to_prdt - SG list to PRTD (Physical Region Description Table, 4DW format)
2580  * @hba:        per-adapter instance
2581  * @lrbp:       pointer to local reference block
2582  * @sg_entries: The number of sg lists actually used
2583  * @sg_list:    Pointer to SG list
2584  */
2585 static void ufshcd_sgl_to_prdt(struct ufs_hba *hba, struct ufshcd_lrb *lrbp, int sg_entries,
2586                                struct scatterlist *sg_list)
2587 {
2588         struct ufshcd_sg_entry *prd;
2589         struct scatterlist *sg;
2590         int i;
2591
2592         if (sg_entries) {
2593
2594                 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
2595                         lrbp->utr_descriptor_ptr->prd_table_length =
2596                                 cpu_to_le16(sg_entries * ufshcd_sg_entry_size(hba));
2597                 else
2598                         lrbp->utr_descriptor_ptr->prd_table_length = cpu_to_le16(sg_entries);
2599
2600                 prd = lrbp->ucd_prdt_ptr;
2601
2602                 for_each_sg(sg_list, sg, sg_entries, i) {
2603                         const unsigned int len = sg_dma_len(sg);
2604
2605                         /*
2606                          * From the UFSHCI spec: "Data Byte Count (DBC): A '0'
2607                          * based value that indicates the length, in bytes, of
2608                          * the data block. A maximum of length of 256KB may
2609                          * exist for any entry. Bits 1:0 of this field shall be
2610                          * 11b to indicate Dword granularity. A value of '3'
2611                          * indicates 4 bytes, '7' indicates 8 bytes, etc."
2612                          */
2613                         WARN_ONCE(len > SZ_256K, "len = %#x\n", len);
2614                         prd->size = cpu_to_le32(len - 1);
2615                         prd->addr = cpu_to_le64(sg->dma_address);
2616                         prd->reserved = 0;
2617                         prd = (void *)prd + ufshcd_sg_entry_size(hba);
2618                 }
2619         } else {
2620                 lrbp->utr_descriptor_ptr->prd_table_length = 0;
2621         }
2622 }
2623
2624 /**
2625  * ufshcd_map_sg - Map scatter-gather list to prdt
2626  * @hba: per adapter instance
2627  * @lrbp: pointer to local reference block
2628  *
2629  * Return: 0 in case of success, non-zero value in case of failure.
2630  */
2631 static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2632 {
2633         struct scsi_cmnd *cmd = lrbp->cmd;
2634         int sg_segments = scsi_dma_map(cmd);
2635
2636         if (sg_segments < 0)
2637                 return sg_segments;
2638
2639         ufshcd_sgl_to_prdt(hba, lrbp, sg_segments, scsi_sglist(cmd));
2640
2641         return ufshcd_crypto_fill_prdt(hba, lrbp);
2642 }
2643
2644 /**
2645  * ufshcd_enable_intr - enable interrupts
2646  * @hba: per adapter instance
2647  * @intrs: interrupt bits
2648  */
2649 static void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs)
2650 {
2651         u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2652
2653         set |= intrs;
2654         ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
2655 }
2656
2657 /**
2658  * ufshcd_disable_intr - disable interrupts
2659  * @hba: per adapter instance
2660  * @intrs: interrupt bits
2661  */
2662 static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs)
2663 {
2664         u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2665
2666         set &= ~intrs;
2667         ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
2668 }
2669
2670 /**
2671  * ufshcd_prepare_req_desc_hdr - Fill UTP Transfer request descriptor header according to request
2672  * descriptor according to request
2673  * @hba: per adapter instance
2674  * @lrbp: pointer to local reference block
2675  * @upiu_flags: flags required in the header
2676  * @cmd_dir: requests data direction
2677  * @ehs_length: Total EHS Length (in 32‐bytes units of all Extra Header Segments)
2678  */
2679 static void
2680 ufshcd_prepare_req_desc_hdr(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
2681                             u8 *upiu_flags, enum dma_data_direction cmd_dir,
2682                             int ehs_length)
2683 {
2684         struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr;
2685         struct request_desc_header *h = &req_desc->header;
2686         enum utp_data_direction data_direction;
2687
2688         lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
2689
2690         *h = (typeof(*h)){ };
2691
2692         if (cmd_dir == DMA_FROM_DEVICE) {
2693                 data_direction = UTP_DEVICE_TO_HOST;
2694                 *upiu_flags = UPIU_CMD_FLAGS_READ;
2695         } else if (cmd_dir == DMA_TO_DEVICE) {
2696                 data_direction = UTP_HOST_TO_DEVICE;
2697                 *upiu_flags = UPIU_CMD_FLAGS_WRITE;
2698         } else {
2699                 data_direction = UTP_NO_DATA_TRANSFER;
2700                 *upiu_flags = UPIU_CMD_FLAGS_NONE;
2701         }
2702
2703         h->command_type = lrbp->command_type;
2704         h->data_direction = data_direction;
2705         h->ehs_length = ehs_length;
2706
2707         if (lrbp->intr_cmd)
2708                 h->interrupt = 1;
2709
2710         /* Prepare crypto related dwords */
2711         ufshcd_prepare_req_desc_hdr_crypto(lrbp, h);
2712
2713         /*
2714          * assigning invalid value for command status. Controller
2715          * updates OCS on command completion, with the command
2716          * status
2717          */
2718         h->ocs = OCS_INVALID_COMMAND_STATUS;
2719
2720         req_desc->prd_table_length = 0;
2721 }
2722
2723 /**
2724  * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc,
2725  * for scsi commands
2726  * @lrbp: local reference block pointer
2727  * @upiu_flags: flags
2728  */
2729 static
2730 void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u8 upiu_flags)
2731 {
2732         struct scsi_cmnd *cmd = lrbp->cmd;
2733         struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2734         unsigned short cdb_len;
2735
2736         ucd_req_ptr->header = (struct utp_upiu_header){
2737                 .transaction_code = UPIU_TRANSACTION_COMMAND,
2738                 .flags = upiu_flags,
2739                 .lun = lrbp->lun,
2740                 .task_tag = lrbp->task_tag,
2741                 .command_set_type = UPIU_COMMAND_SET_TYPE_SCSI,
2742         };
2743
2744         WARN_ON_ONCE(ucd_req_ptr->header.task_tag != lrbp->task_tag);
2745
2746         ucd_req_ptr->sc.exp_data_transfer_len = cpu_to_be32(cmd->sdb.length);
2747
2748         cdb_len = min_t(unsigned short, cmd->cmd_len, UFS_CDB_SIZE);
2749         memcpy(ucd_req_ptr->sc.cdb, cmd->cmnd, cdb_len);
2750
2751         memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2752 }
2753
2754 /**
2755  * ufshcd_prepare_utp_query_req_upiu() - fill the utp_transfer_req_desc for query request
2756  * @hba: UFS hba
2757  * @lrbp: local reference block pointer
2758  * @upiu_flags: flags
2759  */
2760 static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba,
2761                                 struct ufshcd_lrb *lrbp, u8 upiu_flags)
2762 {
2763         struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2764         struct ufs_query *query = &hba->dev_cmd.query;
2765         u16 len = be16_to_cpu(query->request.upiu_req.length);
2766
2767         /* Query request header */
2768         ucd_req_ptr->header = (struct utp_upiu_header){
2769                 .transaction_code = UPIU_TRANSACTION_QUERY_REQ,
2770                 .flags = upiu_flags,
2771                 .lun = lrbp->lun,
2772                 .task_tag = lrbp->task_tag,
2773                 .query_function = query->request.query_func,
2774                 /* Data segment length only need for WRITE_DESC */
2775                 .data_segment_length =
2776                         query->request.upiu_req.opcode ==
2777                                         UPIU_QUERY_OPCODE_WRITE_DESC ?
2778                                 cpu_to_be16(len) :
2779                                 0,
2780         };
2781
2782         /* Copy the Query Request buffer as is */
2783         memcpy(&ucd_req_ptr->qr, &query->request.upiu_req,
2784                         QUERY_OSF_SIZE);
2785
2786         /* Copy the Descriptor */
2787         if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
2788                 memcpy(ucd_req_ptr + 1, query->descriptor, len);
2789
2790         memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2791 }
2792
2793 static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp)
2794 {
2795         struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2796
2797         memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req));
2798
2799         ucd_req_ptr->header = (struct utp_upiu_header){
2800                 .transaction_code = UPIU_TRANSACTION_NOP_OUT,
2801                 .task_tag = lrbp->task_tag,
2802         };
2803
2804         memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2805 }
2806
2807 /**
2808  * ufshcd_compose_devman_upiu - UFS Protocol Information Unit(UPIU)
2809  *                           for Device Management Purposes
2810  * @hba: per adapter instance
2811  * @lrbp: pointer to local reference block
2812  *
2813  * Return: 0 upon success; < 0 upon failure.
2814  */
2815 static int ufshcd_compose_devman_upiu(struct ufs_hba *hba,
2816                                       struct ufshcd_lrb *lrbp)
2817 {
2818         u8 upiu_flags;
2819         int ret = 0;
2820
2821         ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, 0);
2822
2823         if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY)
2824                 ufshcd_prepare_utp_query_req_upiu(hba, lrbp, upiu_flags);
2825         else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP)
2826                 ufshcd_prepare_utp_nop_upiu(lrbp);
2827         else
2828                 ret = -EINVAL;
2829
2830         return ret;
2831 }
2832
2833 /**
2834  * ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU)
2835  *                         for SCSI Purposes
2836  * @hba: per adapter instance
2837  * @lrbp: pointer to local reference block
2838  */
2839 static void ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2840 {
2841         struct request *rq = scsi_cmd_to_rq(lrbp->cmd);
2842         unsigned int ioprio_class = IOPRIO_PRIO_CLASS(req_get_ioprio(rq));
2843         u8 upiu_flags;
2844
2845         ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, lrbp->cmd->sc_data_direction, 0);
2846         if (ioprio_class == IOPRIO_CLASS_RT)
2847                 upiu_flags |= UPIU_CMD_FLAGS_CP;
2848         ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags);
2849 }
2850
2851 static void __ufshcd_setup_cmd(struct ufshcd_lrb *lrbp, struct scsi_cmnd *cmd, u8 lun, int tag)
2852 {
2853         memset(lrbp->ucd_req_ptr, 0, sizeof(*lrbp->ucd_req_ptr));
2854
2855         lrbp->cmd = cmd;
2856         lrbp->task_tag = tag;
2857         lrbp->lun = lun;
2858         ufshcd_prepare_lrbp_crypto(cmd ? scsi_cmd_to_rq(cmd) : NULL, lrbp);
2859 }
2860
2861 static void ufshcd_setup_scsi_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
2862                                   struct scsi_cmnd *cmd, u8 lun, int tag)
2863 {
2864         __ufshcd_setup_cmd(lrbp, cmd, lun, tag);
2865         lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba);
2866         lrbp->req_abort_skip = false;
2867
2868         ufshcd_comp_scsi_upiu(hba, lrbp);
2869 }
2870
2871 /**
2872  * ufshcd_upiu_wlun_to_scsi_wlun - maps UPIU W-LUN id to SCSI W-LUN ID
2873  * @upiu_wlun_id: UPIU W-LUN id
2874  *
2875  * Return: SCSI W-LUN id.
2876  */
2877 static inline u16 ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id)
2878 {
2879         return (upiu_wlun_id & ~UFS_UPIU_WLUN_ID) | SCSI_W_LUN_BASE;
2880 }
2881
2882 static inline bool is_device_wlun(struct scsi_device *sdev)
2883 {
2884         return sdev->lun ==
2885                 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN);
2886 }
2887
2888 /*
2889  * Associate the UFS controller queue with the default and poll HCTX types.
2890  * Initialize the mq_map[] arrays.
2891  */
2892 static void ufshcd_map_queues(struct Scsi_Host *shost)
2893 {
2894         struct ufs_hba *hba = shost_priv(shost);
2895         int i, queue_offset = 0;
2896
2897         if (!is_mcq_supported(hba)) {
2898                 hba->nr_queues[HCTX_TYPE_DEFAULT] = 1;
2899                 hba->nr_queues[HCTX_TYPE_READ] = 0;
2900                 hba->nr_queues[HCTX_TYPE_POLL] = 1;
2901                 hba->nr_hw_queues = 1;
2902         }
2903
2904         for (i = 0; i < shost->nr_maps; i++) {
2905                 struct blk_mq_queue_map *map = &shost->tag_set.map[i];
2906
2907                 map->nr_queues = hba->nr_queues[i];
2908                 if (!map->nr_queues)
2909                         continue;
2910                 map->queue_offset = queue_offset;
2911                 if (i == HCTX_TYPE_POLL && !is_mcq_supported(hba))
2912                         map->queue_offset = 0;
2913
2914                 blk_mq_map_queues(map);
2915                 queue_offset += map->nr_queues;
2916         }
2917 }
2918
2919 static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb *lrb, int i)
2920 {
2921         struct utp_transfer_cmd_desc *cmd_descp = (void *)hba->ucdl_base_addr +
2922                 i * ufshcd_get_ucd_size(hba);
2923         struct utp_transfer_req_desc *utrdlp = hba->utrdl_base_addr;
2924         dma_addr_t cmd_desc_element_addr = hba->ucdl_dma_addr +
2925                 i * ufshcd_get_ucd_size(hba);
2926         u16 response_offset = le16_to_cpu(utrdlp[i].response_upiu_offset);
2927         u16 prdt_offset = le16_to_cpu(utrdlp[i].prd_table_offset);
2928
2929         lrb->utr_descriptor_ptr = utrdlp + i;
2930         lrb->utrd_dma_addr = hba->utrdl_dma_addr +
2931                 i * sizeof(struct utp_transfer_req_desc);
2932         lrb->ucd_req_ptr = (struct utp_upiu_req *)cmd_descp->command_upiu;
2933         lrb->ucd_req_dma_addr = cmd_desc_element_addr;
2934         lrb->ucd_rsp_ptr = (struct utp_upiu_rsp *)cmd_descp->response_upiu;
2935         lrb->ucd_rsp_dma_addr = cmd_desc_element_addr + response_offset;
2936         lrb->ucd_prdt_ptr = (struct ufshcd_sg_entry *)cmd_descp->prd_table;
2937         lrb->ucd_prdt_dma_addr = cmd_desc_element_addr + prdt_offset;
2938 }
2939
2940 /**
2941  * ufshcd_queuecommand - main entry point for SCSI requests
2942  * @host: SCSI host pointer
2943  * @cmd: command from SCSI Midlayer
2944  *
2945  * Return: 0 for success, non-zero in case of failure.
2946  */
2947 static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
2948 {
2949         struct ufs_hba *hba = shost_priv(host);
2950         int tag = scsi_cmd_to_rq(cmd)->tag;
2951         struct ufshcd_lrb *lrbp;
2952         int err = 0;
2953         struct ufs_hw_queue *hwq = NULL;
2954
2955         switch (hba->ufshcd_state) {
2956         case UFSHCD_STATE_OPERATIONAL:
2957                 break;
2958         case UFSHCD_STATE_EH_SCHEDULED_NON_FATAL:
2959                 /*
2960                  * SCSI error handler can call ->queuecommand() while UFS error
2961                  * handler is in progress. Error interrupts could change the
2962                  * state from UFSHCD_STATE_RESET to
2963                  * UFSHCD_STATE_EH_SCHEDULED_NON_FATAL. Prevent requests
2964                  * being issued in that case.
2965                  */
2966                 if (ufshcd_eh_in_progress(hba)) {
2967                         err = SCSI_MLQUEUE_HOST_BUSY;
2968                         goto out;
2969                 }
2970                 break;
2971         case UFSHCD_STATE_EH_SCHEDULED_FATAL:
2972                 /*
2973                  * pm_runtime_get_sync() is used at error handling preparation
2974                  * stage. If a scsi cmd, e.g. the SSU cmd, is sent from hba's
2975                  * PM ops, it can never be finished if we let SCSI layer keep
2976                  * retrying it, which gets err handler stuck forever. Neither
2977                  * can we let the scsi cmd pass through, because UFS is in bad
2978                  * state, the scsi cmd may eventually time out, which will get
2979                  * err handler blocked for too long. So, just fail the scsi cmd
2980                  * sent from PM ops, err handler can recover PM error anyways.
2981                  */
2982                 if (hba->pm_op_in_progress) {
2983                         hba->force_reset = true;
2984                         set_host_byte(cmd, DID_BAD_TARGET);
2985                         scsi_done(cmd);
2986                         goto out;
2987                 }
2988                 fallthrough;
2989         case UFSHCD_STATE_RESET:
2990                 err = SCSI_MLQUEUE_HOST_BUSY;
2991                 goto out;
2992         case UFSHCD_STATE_ERROR:
2993                 set_host_byte(cmd, DID_ERROR);
2994                 scsi_done(cmd);
2995                 goto out;
2996         }
2997
2998         hba->req_abort_count = 0;
2999
3000         ufshcd_hold(hba);
3001
3002         lrbp = &hba->lrb[tag];
3003
3004         ufshcd_setup_scsi_cmd(hba, lrbp, cmd, ufshcd_scsi_to_upiu_lun(cmd->device->lun), tag);
3005
3006         err = ufshcd_map_sg(hba, lrbp);
3007         if (err) {
3008                 ufshcd_release(hba);
3009                 goto out;
3010         }
3011
3012         if (hba->mcq_enabled)
3013                 hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd));
3014
3015         ufshcd_send_command(hba, tag, hwq);
3016
3017 out:
3018         if (ufs_trigger_eh(hba)) {
3019                 unsigned long flags;
3020
3021                 spin_lock_irqsave(hba->host->host_lock, flags);
3022                 ufshcd_schedule_eh_work(hba);
3023                 spin_unlock_irqrestore(hba->host->host_lock, flags);
3024         }
3025
3026         return err;
3027 }
3028
3029 static void ufshcd_setup_dev_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
3030                              enum dev_cmd_type cmd_type, u8 lun, int tag)
3031 {
3032         __ufshcd_setup_cmd(lrbp, NULL, lun, tag);
3033         lrbp->intr_cmd = true; /* No interrupt aggregation */
3034         hba->dev_cmd.type = cmd_type;
3035 }
3036
3037 static int ufshcd_compose_dev_cmd(struct ufs_hba *hba,
3038                 struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag)
3039 {
3040         ufshcd_setup_dev_cmd(hba, lrbp, cmd_type, 0, tag);
3041
3042         return ufshcd_compose_devman_upiu(hba, lrbp);
3043 }
3044
3045 /*
3046  * Check with the block layer if the command is inflight
3047  * @cmd: command to check.
3048  *
3049  * Return: true if command is inflight; false if not.
3050  */
3051 bool ufshcd_cmd_inflight(struct scsi_cmnd *cmd)
3052 {
3053         return cmd && blk_mq_rq_state(scsi_cmd_to_rq(cmd)) == MQ_RQ_IN_FLIGHT;
3054 }
3055
3056 /*
3057  * Clear the pending command in the controller and wait until
3058  * the controller confirms that the command has been cleared.
3059  * @hba: per adapter instance
3060  * @task_tag: The tag number of the command to be cleared.
3061  */
3062 static int ufshcd_clear_cmd(struct ufs_hba *hba, u32 task_tag)
3063 {
3064         u32 mask;
3065         int err;
3066
3067         if (hba->mcq_enabled) {
3068                 /*
3069                  * MCQ mode. Clean up the MCQ resources similar to
3070                  * what the ufshcd_utrl_clear() does for SDB mode.
3071                  */
3072                 err = ufshcd_mcq_sq_cleanup(hba, task_tag);
3073                 if (err) {
3074                         dev_err(hba->dev, "%s: failed tag=%d. err=%d\n",
3075                                 __func__, task_tag, err);
3076                         return err;
3077                 }
3078                 return 0;
3079         }
3080
3081         mask = 1U << task_tag;
3082
3083         /* clear outstanding transaction before retry */
3084         ufshcd_utrl_clear(hba, mask);
3085
3086         /*
3087          * wait for h/w to clear corresponding bit in door-bell.
3088          * max. wait is 1 sec.
3089          */
3090         return ufshcd_wait_for_register(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL,
3091                                         mask, ~mask, 1000, 1000);
3092 }
3093
3094 /**
3095  * ufshcd_dev_cmd_completion() - handles device management command responses
3096  * @hba: per adapter instance
3097  * @lrbp: pointer to local reference block
3098  *
3099  * Return: 0 upon success; < 0 upon failure.
3100  */
3101 static int
3102 ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
3103 {
3104         enum upiu_response_transaction resp;
3105         int err = 0;
3106
3107         hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
3108         resp = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
3109
3110         switch (resp) {
3111         case UPIU_TRANSACTION_NOP_IN:
3112                 if (hba->dev_cmd.type != DEV_CMD_TYPE_NOP) {
3113                         err = -EINVAL;
3114                         dev_err(hba->dev, "%s: unexpected response %x\n",
3115                                         __func__, resp);
3116                 }
3117                 break;
3118         case UPIU_TRANSACTION_QUERY_RSP: {
3119                 u8 response = lrbp->ucd_rsp_ptr->header.response;
3120
3121                 if (response == 0)
3122                         err = ufshcd_copy_query_response(hba, lrbp);
3123                 break;
3124         }
3125         case UPIU_TRANSACTION_REJECT_UPIU:
3126                 /* TODO: handle Reject UPIU Response */
3127                 err = -EPERM;
3128                 dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n",
3129                                 __func__);
3130                 break;
3131         case UPIU_TRANSACTION_RESPONSE:
3132                 if (hba->dev_cmd.type != DEV_CMD_TYPE_RPMB) {
3133                         err = -EINVAL;
3134                         dev_err(hba->dev, "%s: unexpected response %x\n", __func__, resp);
3135                 }
3136                 break;
3137         default:
3138                 err = -EINVAL;
3139                 dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n",
3140                                 __func__, resp);
3141                 break;
3142         }
3143
3144         return err;
3145 }
3146
3147 static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba,
3148                 struct ufshcd_lrb *lrbp, int max_timeout)
3149 {
3150         unsigned long time_left = msecs_to_jiffies(max_timeout);
3151         unsigned long flags;
3152         bool pending;
3153         int err;
3154
3155 retry:
3156         time_left = wait_for_completion_timeout(hba->dev_cmd.complete,
3157                                                 time_left);
3158
3159         if (likely(time_left)) {
3160                 /*
3161                  * The completion handler called complete() and the caller of
3162                  * this function still owns the @lrbp tag so the code below does
3163                  * not trigger any race conditions.
3164                  */
3165                 hba->dev_cmd.complete = NULL;
3166                 err = ufshcd_get_tr_ocs(lrbp, NULL);
3167                 if (!err)
3168                         err = ufshcd_dev_cmd_completion(hba, lrbp);
3169         } else {
3170                 err = -ETIMEDOUT;
3171                 dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n",
3172                         __func__, lrbp->task_tag);
3173
3174                 /* MCQ mode */
3175                 if (hba->mcq_enabled) {
3176                         /* successfully cleared the command, retry if needed */
3177                         if (ufshcd_clear_cmd(hba, lrbp->task_tag) == 0)
3178                                 err = -EAGAIN;
3179                         hba->dev_cmd.complete = NULL;
3180                         return err;
3181                 }
3182
3183                 /* SDB mode */
3184                 if (ufshcd_clear_cmd(hba, lrbp->task_tag) == 0) {
3185                         /* successfully cleared the command, retry if needed */
3186                         err = -EAGAIN;
3187                         /*
3188                          * Since clearing the command succeeded we also need to
3189                          * clear the task tag bit from the outstanding_reqs
3190                          * variable.
3191                          */
3192                         spin_lock_irqsave(&hba->outstanding_lock, flags);
3193                         pending = test_bit(lrbp->task_tag,
3194                                            &hba->outstanding_reqs);
3195                         if (pending) {
3196                                 hba->dev_cmd.complete = NULL;
3197                                 __clear_bit(lrbp->task_tag,
3198                                             &hba->outstanding_reqs);
3199                         }
3200                         spin_unlock_irqrestore(&hba->outstanding_lock, flags);
3201
3202                         if (!pending) {
3203                                 /*
3204                                  * The completion handler ran while we tried to
3205                                  * clear the command.
3206                                  */
3207                                 time_left = 1;
3208                                 goto retry;
3209                         }
3210                 } else {
3211                         dev_err(hba->dev, "%s: failed to clear tag %d\n",
3212                                 __func__, lrbp->task_tag);
3213
3214                         spin_lock_irqsave(&hba->outstanding_lock, flags);
3215                         pending = test_bit(lrbp->task_tag,
3216                                            &hba->outstanding_reqs);
3217                         if (pending)
3218                                 hba->dev_cmd.complete = NULL;
3219                         spin_unlock_irqrestore(&hba->outstanding_lock, flags);
3220
3221                         if (!pending) {
3222                                 /*
3223                                  * The completion handler ran while we tried to
3224                                  * clear the command.
3225                                  */
3226                                 time_left = 1;
3227                                 goto retry;
3228                         }
3229                 }
3230         }
3231
3232         return err;
3233 }
3234
3235 static void ufshcd_dev_man_lock(struct ufs_hba *hba)
3236 {
3237         ufshcd_hold(hba);
3238         mutex_lock(&hba->dev_cmd.lock);
3239         down_read(&hba->clk_scaling_lock);
3240 }
3241
3242 static void ufshcd_dev_man_unlock(struct ufs_hba *hba)
3243 {
3244         up_read(&hba->clk_scaling_lock);
3245         mutex_unlock(&hba->dev_cmd.lock);
3246         ufshcd_release(hba);
3247 }
3248
3249 static int ufshcd_issue_dev_cmd(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
3250                           const u32 tag, int timeout)
3251 {
3252         DECLARE_COMPLETION_ONSTACK(wait);
3253         int err;
3254
3255         hba->dev_cmd.complete = &wait;
3256
3257         ufshcd_add_query_upiu_trace(hba, UFS_QUERY_SEND, lrbp->ucd_req_ptr);
3258
3259         ufshcd_send_command(hba, tag, hba->dev_cmd_queue);
3260         err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout);
3261
3262         ufshcd_add_query_upiu_trace(hba, err ? UFS_QUERY_ERR : UFS_QUERY_COMP,
3263                                     (struct utp_upiu_req *)lrbp->ucd_rsp_ptr);
3264
3265         return err;
3266 }
3267
3268 /**
3269  * ufshcd_exec_dev_cmd - API for sending device management requests
3270  * @hba: UFS hba
3271  * @cmd_type: specifies the type (NOP, Query...)
3272  * @timeout: timeout in milliseconds
3273  *
3274  * Return: 0 upon success; < 0 upon failure.
3275  *
3276  * NOTE: Since there is only one available tag for device management commands,
3277  * it is expected you hold the hba->dev_cmd.lock mutex.
3278  */
3279 static int ufshcd_exec_dev_cmd(struct ufs_hba *hba,
3280                 enum dev_cmd_type cmd_type, int timeout)
3281 {
3282         const u32 tag = hba->reserved_slot;
3283         struct ufshcd_lrb *lrbp = &hba->lrb[tag];
3284         int err;
3285
3286         /* Protects use of hba->reserved_slot. */
3287         lockdep_assert_held(&hba->dev_cmd.lock);
3288
3289         err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag);
3290         if (unlikely(err))
3291                 return err;
3292
3293         return ufshcd_issue_dev_cmd(hba, lrbp, tag, timeout);
3294 }
3295
3296 /**
3297  * ufshcd_init_query() - init the query response and request parameters
3298  * @hba: per-adapter instance
3299  * @request: address of the request pointer to be initialized
3300  * @response: address of the response pointer to be initialized
3301  * @opcode: operation to perform
3302  * @idn: flag idn to access
3303  * @index: LU number to access
3304  * @selector: query/flag/descriptor further identification
3305  */
3306 static inline void ufshcd_init_query(struct ufs_hba *hba,
3307                 struct ufs_query_req **request, struct ufs_query_res **response,
3308                 enum query_opcode opcode, u8 idn, u8 index, u8 selector)
3309 {
3310         *request = &hba->dev_cmd.query.request;
3311         *response = &hba->dev_cmd.query.response;
3312         memset(*request, 0, sizeof(struct ufs_query_req));
3313         memset(*response, 0, sizeof(struct ufs_query_res));
3314         (*request)->upiu_req.opcode = opcode;
3315         (*request)->upiu_req.idn = idn;
3316         (*request)->upiu_req.index = index;
3317         (*request)->upiu_req.selector = selector;
3318 }
3319
3320 static int ufshcd_query_flag_retry(struct ufs_hba *hba,
3321         enum query_opcode opcode, enum flag_idn idn, u8 index, bool *flag_res)
3322 {
3323         int ret;
3324         int retries;
3325
3326         for (retries = 0; retries < QUERY_REQ_RETRIES; retries++) {
3327                 ret = ufshcd_query_flag(hba, opcode, idn, index, flag_res);
3328                 if (ret)
3329                         dev_dbg(hba->dev,
3330                                 "%s: failed with error %d, retries %d\n",
3331                                 __func__, ret, retries);
3332                 else
3333                         break;
3334         }
3335
3336         if (ret)
3337                 dev_err(hba->dev,
3338                         "%s: query flag, opcode %d, idn %d, failed with error %d after %d retries\n",
3339                         __func__, opcode, idn, ret, retries);
3340         return ret;
3341 }
3342
3343 /**
3344  * ufshcd_query_flag() - API function for sending flag query requests
3345  * @hba: per-adapter instance
3346  * @opcode: flag query to perform
3347  * @idn: flag idn to access
3348  * @index: flag index to access
3349  * @flag_res: the flag value after the query request completes
3350  *
3351  * Return: 0 for success, non-zero in case of failure.
3352  */
3353 int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode,
3354                         enum flag_idn idn, u8 index, bool *flag_res)
3355 {
3356         struct ufs_query_req *request = NULL;
3357         struct ufs_query_res *response = NULL;
3358         int err, selector = 0;
3359         int timeout = QUERY_REQ_TIMEOUT;
3360
3361         BUG_ON(!hba);
3362
3363         ufshcd_dev_man_lock(hba);
3364
3365         ufshcd_init_query(hba, &request, &response, opcode, idn, index,
3366                         selector);
3367
3368         switch (opcode) {
3369         case UPIU_QUERY_OPCODE_SET_FLAG:
3370         case UPIU_QUERY_OPCODE_CLEAR_FLAG:
3371         case UPIU_QUERY_OPCODE_TOGGLE_FLAG:
3372                 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
3373                 break;
3374         case UPIU_QUERY_OPCODE_READ_FLAG:
3375                 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
3376                 if (!flag_res) {
3377                         /* No dummy reads */
3378                         dev_err(hba->dev, "%s: Invalid argument for read request\n",
3379                                         __func__);
3380                         err = -EINVAL;
3381                         goto out_unlock;
3382                 }
3383                 break;
3384         default:
3385                 dev_err(hba->dev,
3386                         "%s: Expected query flag opcode but got = %d\n",
3387                         __func__, opcode);
3388                 err = -EINVAL;
3389                 goto out_unlock;
3390         }
3391
3392         err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, timeout);
3393
3394         if (err) {
3395                 dev_err(hba->dev,
3396                         "%s: Sending flag query for idn %d failed, err = %d\n",
3397                         __func__, idn, err);
3398                 goto out_unlock;
3399         }
3400
3401         if (flag_res)
3402                 *flag_res = (be32_to_cpu(response->upiu_res.value) &
3403                                 MASK_QUERY_UPIU_FLAG_LOC) & 0x1;
3404
3405 out_unlock:
3406         ufshcd_dev_man_unlock(hba);
3407         return err;
3408 }
3409
3410 /**
3411  * ufshcd_query_attr - API function for sending attribute requests
3412  * @hba: per-adapter instance
3413  * @opcode: attribute opcode
3414  * @idn: attribute idn to access
3415  * @index: index field
3416  * @selector: selector field
3417  * @attr_val: the attribute value after the query request completes
3418  *
3419  * Return: 0 for success, non-zero in case of failure.
3420 */
3421 int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode,
3422                       enum attr_idn idn, u8 index, u8 selector, u32 *attr_val)
3423 {
3424         struct ufs_query_req *request = NULL;
3425         struct ufs_query_res *response = NULL;
3426         int err;
3427
3428         BUG_ON(!hba);
3429
3430         if (!attr_val) {
3431                 dev_err(hba->dev, "%s: attribute value required for opcode 0x%x\n",
3432                                 __func__, opcode);
3433                 return -EINVAL;
3434         }
3435
3436         ufshcd_dev_man_lock(hba);
3437
3438         ufshcd_init_query(hba, &request, &response, opcode, idn, index,
3439                         selector);
3440
3441         switch (opcode) {
3442         case UPIU_QUERY_OPCODE_WRITE_ATTR:
3443                 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
3444                 request->upiu_req.value = cpu_to_be32(*attr_val);
3445                 break;
3446         case UPIU_QUERY_OPCODE_READ_ATTR:
3447                 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
3448                 break;
3449         default:
3450                 dev_err(hba->dev, "%s: Expected query attr opcode but got = 0x%.2x\n",
3451                                 __func__, opcode);
3452                 err = -EINVAL;
3453                 goto out_unlock;
3454         }
3455
3456         err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
3457
3458         if (err) {
3459                 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
3460                                 __func__, opcode, idn, index, err);
3461                 goto out_unlock;
3462         }
3463
3464         *attr_val = be32_to_cpu(response->upiu_res.value);
3465
3466 out_unlock:
3467         ufshcd_dev_man_unlock(hba);
3468         return err;
3469 }
3470
3471 /**
3472  * ufshcd_query_attr_retry() - API function for sending query
3473  * attribute with retries
3474  * @hba: per-adapter instance
3475  * @opcode: attribute opcode
3476  * @idn: attribute idn to access
3477  * @index: index field
3478  * @selector: selector field
3479  * @attr_val: the attribute value after the query request
3480  * completes
3481  *
3482  * Return: 0 for success, non-zero in case of failure.
3483 */
3484 int ufshcd_query_attr_retry(struct ufs_hba *hba,
3485         enum query_opcode opcode, enum attr_idn idn, u8 index, u8 selector,
3486         u32 *attr_val)
3487 {
3488         int ret = 0;
3489         u32 retries;
3490
3491         for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
3492                 ret = ufshcd_query_attr(hba, opcode, idn, index,
3493                                                 selector, attr_val);
3494                 if (ret)
3495                         dev_dbg(hba->dev, "%s: failed with error %d, retries %d\n",
3496                                 __func__, ret, retries);
3497                 else
3498                         break;
3499         }
3500
3501         if (ret)
3502                 dev_err(hba->dev,
3503                         "%s: query attribute, idn %d, failed with error %d after %d retries\n",
3504                         __func__, idn, ret, QUERY_REQ_RETRIES);
3505         return ret;
3506 }
3507
3508 static int __ufshcd_query_descriptor(struct ufs_hba *hba,
3509                         enum query_opcode opcode, enum desc_idn idn, u8 index,
3510                         u8 selector, u8 *desc_buf, int *buf_len)
3511 {
3512         struct ufs_query_req *request = NULL;
3513         struct ufs_query_res *response = NULL;
3514         int err;
3515
3516         BUG_ON(!hba);
3517
3518         if (!desc_buf) {
3519                 dev_err(hba->dev, "%s: descriptor buffer required for opcode 0x%x\n",
3520                                 __func__, opcode);
3521                 return -EINVAL;
3522         }
3523
3524         if (*buf_len < QUERY_DESC_MIN_SIZE || *buf_len > QUERY_DESC_MAX_SIZE) {
3525                 dev_err(hba->dev, "%s: descriptor buffer size (%d) is out of range\n",
3526                                 __func__, *buf_len);
3527                 return -EINVAL;
3528         }
3529
3530         ufshcd_dev_man_lock(hba);
3531
3532         ufshcd_init_query(hba, &request, &response, opcode, idn, index,
3533                         selector);
3534         hba->dev_cmd.query.descriptor = desc_buf;
3535         request->upiu_req.length = cpu_to_be16(*buf_len);
3536
3537         switch (opcode) {
3538         case UPIU_QUERY_OPCODE_WRITE_DESC:
3539                 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
3540                 break;
3541         case UPIU_QUERY_OPCODE_READ_DESC:
3542                 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
3543                 break;
3544         default:
3545                 dev_err(hba->dev,
3546                                 "%s: Expected query descriptor opcode but got = 0x%.2x\n",
3547                                 __func__, opcode);
3548                 err = -EINVAL;
3549                 goto out_unlock;
3550         }
3551
3552         err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
3553
3554         if (err) {
3555                 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
3556                                 __func__, opcode, idn, index, err);
3557                 goto out_unlock;
3558         }
3559
3560         *buf_len = be16_to_cpu(response->upiu_res.length);
3561
3562 out_unlock:
3563         hba->dev_cmd.query.descriptor = NULL;
3564         ufshcd_dev_man_unlock(hba);
3565         return err;
3566 }
3567
3568 /**
3569  * ufshcd_query_descriptor_retry - API function for sending descriptor requests
3570  * @hba: per-adapter instance
3571  * @opcode: attribute opcode
3572  * @idn: attribute idn to access
3573  * @index: index field
3574  * @selector: selector field
3575  * @desc_buf: the buffer that contains the descriptor
3576  * @buf_len: length parameter passed to the device
3577  *
3578  * The buf_len parameter will contain, on return, the length parameter
3579  * received on the response.
3580  *
3581  * Return: 0 for success, non-zero in case of failure.
3582  */
3583 int ufshcd_query_descriptor_retry(struct ufs_hba *hba,
3584                                   enum query_opcode opcode,
3585                                   enum desc_idn idn, u8 index,
3586                                   u8 selector,
3587                                   u8 *desc_buf, int *buf_len)
3588 {
3589         int err;
3590         int retries;
3591
3592         for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
3593                 err = __ufshcd_query_descriptor(hba, opcode, idn, index,
3594                                                 selector, desc_buf, buf_len);
3595                 if (!err || err == -EINVAL)
3596                         break;
3597         }
3598
3599         return err;
3600 }
3601
3602 /**
3603  * ufshcd_read_desc_param - read the specified descriptor parameter
3604  * @hba: Pointer to adapter instance
3605  * @desc_id: descriptor idn value
3606  * @desc_index: descriptor index
3607  * @param_offset: offset of the parameter to read
3608  * @param_read_buf: pointer to buffer where parameter would be read
3609  * @param_size: sizeof(param_read_buf)
3610  *
3611  * Return: 0 in case of success, non-zero otherwise.
3612  */
3613 int ufshcd_read_desc_param(struct ufs_hba *hba,
3614                            enum desc_idn desc_id,
3615                            int desc_index,
3616                            u8 param_offset,
3617                            u8 *param_read_buf,
3618                            u8 param_size)
3619 {
3620         int ret;
3621         u8 *desc_buf;
3622         int buff_len = QUERY_DESC_MAX_SIZE;
3623         bool is_kmalloc = true;
3624
3625         /* Safety check */
3626         if (desc_id >= QUERY_DESC_IDN_MAX || !param_size)
3627                 return -EINVAL;
3628
3629         /* Check whether we need temp memory */
3630         if (param_offset != 0 || param_size < buff_len) {
3631                 desc_buf = kzalloc(buff_len, GFP_KERNEL);
3632                 if (!desc_buf)
3633                         return -ENOMEM;
3634         } else {
3635                 desc_buf = param_read_buf;
3636                 is_kmalloc = false;
3637         }
3638
3639         /* Request for full descriptor */
3640         ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC,
3641                                             desc_id, desc_index, 0,
3642                                             desc_buf, &buff_len);
3643         if (ret) {
3644                 dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d, desc_index %d, param_offset %d, ret %d\n",
3645                         __func__, desc_id, desc_index, param_offset, ret);
3646                 goto out;
3647         }
3648
3649         /* Update descriptor length */
3650         buff_len = desc_buf[QUERY_DESC_LENGTH_OFFSET];
3651
3652         if (param_offset >= buff_len) {
3653                 dev_err(hba->dev, "%s: Invalid offset 0x%x in descriptor IDN 0x%x, length 0x%x\n",
3654                         __func__, param_offset, desc_id, buff_len);
3655                 ret = -EINVAL;
3656                 goto out;
3657         }
3658
3659         /* Sanity check */
3660         if (desc_buf[QUERY_DESC_DESC_TYPE_OFFSET] != desc_id) {
3661                 dev_err(hba->dev, "%s: invalid desc_id %d in descriptor header\n",
3662                         __func__, desc_buf[QUERY_DESC_DESC_TYPE_OFFSET]);
3663                 ret = -EINVAL;
3664                 goto out;
3665         }
3666
3667         if (is_kmalloc) {
3668                 /* Make sure we don't copy more data than available */
3669                 if (param_offset >= buff_len)
3670                         ret = -EINVAL;
3671                 else
3672                         memcpy(param_read_buf, &desc_buf[param_offset],
3673                                min_t(u32, param_size, buff_len - param_offset));
3674         }
3675 out:
3676         if (is_kmalloc)
3677                 kfree(desc_buf);
3678         return ret;
3679 }
3680
3681 /**
3682  * struct uc_string_id - unicode string
3683  *
3684  * @len: size of this descriptor inclusive
3685  * @type: descriptor type
3686  * @uc: unicode string character
3687  */
3688 struct uc_string_id {
3689         u8 len;
3690         u8 type;
3691         wchar_t uc[];
3692 } __packed;
3693
3694 /* replace non-printable or non-ASCII characters with spaces */
3695 static inline char ufshcd_remove_non_printable(u8 ch)
3696 {
3697         return (ch >= 0x20 && ch <= 0x7e) ? ch : ' ';
3698 }
3699
3700 /**
3701  * ufshcd_read_string_desc - read string descriptor
3702  * @hba: pointer to adapter instance
3703  * @desc_index: descriptor index
3704  * @buf: pointer to buffer where descriptor would be read,
3705  *       the caller should free the memory.
3706  * @ascii: if true convert from unicode to ascii characters
3707  *         null terminated string.
3708  *
3709  * Return:
3710  * *      string size on success.
3711  * *      -ENOMEM: on allocation failure
3712  * *      -EINVAL: on a wrong parameter
3713  */
3714 int ufshcd_read_string_desc(struct ufs_hba *hba, u8 desc_index,
3715                             u8 **buf, bool ascii)
3716 {
3717         struct uc_string_id *uc_str;
3718         u8 *str;
3719         int ret;
3720
3721         if (!buf)
3722                 return -EINVAL;
3723
3724         uc_str = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL);
3725         if (!uc_str)
3726                 return -ENOMEM;
3727
3728         ret = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_STRING, desc_index, 0,
3729                                      (u8 *)uc_str, QUERY_DESC_MAX_SIZE);
3730         if (ret < 0) {
3731                 dev_err(hba->dev, "Reading String Desc failed after %d retries. err = %d\n",
3732                         QUERY_REQ_RETRIES, ret);
3733                 str = NULL;
3734                 goto out;
3735         }
3736
3737         if (uc_str->len <= QUERY_DESC_HDR_SIZE) {
3738                 dev_dbg(hba->dev, "String Desc is of zero length\n");
3739                 str = NULL;
3740                 ret = 0;
3741                 goto out;
3742         }
3743
3744         if (ascii) {
3745                 ssize_t ascii_len;
3746                 int i;
3747                 /* remove header and divide by 2 to move from UTF16 to UTF8 */
3748                 ascii_len = (uc_str->len - QUERY_DESC_HDR_SIZE) / 2 + 1;
3749                 str = kzalloc(ascii_len, GFP_KERNEL);
3750                 if (!str) {
3751                         ret = -ENOMEM;
3752                         goto out;
3753                 }
3754
3755                 /*
3756                  * the descriptor contains string in UTF16 format
3757                  * we need to convert to utf-8 so it can be displayed
3758                  */
3759                 ret = utf16s_to_utf8s(uc_str->uc,
3760                                       uc_str->len - QUERY_DESC_HDR_SIZE,
3761                                       UTF16_BIG_ENDIAN, str, ascii_len - 1);
3762
3763                 /* replace non-printable or non-ASCII characters with spaces */
3764                 for (i = 0; i < ret; i++)
3765                         str[i] = ufshcd_remove_non_printable(str[i]);
3766
3767                 str[ret++] = '\0';
3768
3769         } else {
3770                 str = kmemdup(uc_str, uc_str->len, GFP_KERNEL);
3771                 if (!str) {
3772                         ret = -ENOMEM;
3773                         goto out;
3774                 }
3775                 ret = uc_str->len;
3776         }
3777 out:
3778         *buf = str;
3779         kfree(uc_str);
3780         return ret;
3781 }
3782
3783 /**
3784  * ufshcd_read_unit_desc_param - read the specified unit descriptor parameter
3785  * @hba: Pointer to adapter instance
3786  * @lun: lun id
3787  * @param_offset: offset of the parameter to read
3788  * @param_read_buf: pointer to buffer where parameter would be read
3789  * @param_size: sizeof(param_read_buf)
3790  *
3791  * Return: 0 in case of success, non-zero otherwise.
3792  */
3793 static inline int ufshcd_read_unit_desc_param(struct ufs_hba *hba,
3794                                               int lun,
3795                                               enum unit_desc_param param_offset,
3796                                               u8 *param_read_buf,
3797                                               u32 param_size)
3798 {
3799         /*
3800          * Unit descriptors are only available for general purpose LUs (LUN id
3801          * from 0 to 7) and RPMB Well known LU.
3802          */
3803         if (!ufs_is_valid_unit_desc_lun(&hba->dev_info, lun))
3804                 return -EOPNOTSUPP;
3805
3806         return ufshcd_read_desc_param(hba, QUERY_DESC_IDN_UNIT, lun,
3807                                       param_offset, param_read_buf, param_size);
3808 }
3809
3810 static int ufshcd_get_ref_clk_gating_wait(struct ufs_hba *hba)
3811 {
3812         int err = 0;
3813         u32 gating_wait = UFSHCD_REF_CLK_GATING_WAIT_US;
3814
3815         if (hba->dev_info.wspecversion >= 0x300) {
3816                 err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
3817                                 QUERY_ATTR_IDN_REF_CLK_GATING_WAIT_TIME, 0, 0,
3818                                 &gating_wait);
3819                 if (err)
3820                         dev_err(hba->dev, "Failed reading bRefClkGatingWait. err = %d, use default %uus\n",
3821                                          err, gating_wait);
3822
3823                 if (gating_wait == 0) {
3824                         gating_wait = UFSHCD_REF_CLK_GATING_WAIT_US;
3825                         dev_err(hba->dev, "Undefined ref clk gating wait time, use default %uus\n",
3826                                          gating_wait);
3827                 }
3828
3829                 hba->dev_info.clk_gating_wait_us = gating_wait;
3830         }
3831
3832         return err;
3833 }
3834
3835 /**
3836  * ufshcd_memory_alloc - allocate memory for host memory space data structures
3837  * @hba: per adapter instance
3838  *
3839  * 1. Allocate DMA memory for Command Descriptor array
3840  *      Each command descriptor consist of Command UPIU, Response UPIU and PRDT
3841  * 2. Allocate DMA memory for UTP Transfer Request Descriptor List (UTRDL).
3842  * 3. Allocate DMA memory for UTP Task Management Request Descriptor List
3843  *      (UTMRDL)
3844  * 4. Allocate memory for local reference block(lrb).
3845  *
3846  * Return: 0 for success, non-zero in case of failure.
3847  */
3848 static int ufshcd_memory_alloc(struct ufs_hba *hba)
3849 {
3850         size_t utmrdl_size, utrdl_size, ucdl_size;
3851
3852         /* Allocate memory for UTP command descriptors */
3853         ucdl_size = ufshcd_get_ucd_size(hba) * hba->nutrs;
3854         hba->ucdl_base_addr = dmam_alloc_coherent(hba->dev,
3855                                                   ucdl_size,
3856                                                   &hba->ucdl_dma_addr,
3857                                                   GFP_KERNEL);
3858
3859         /*
3860          * UFSHCI requires UTP command descriptor to be 128 byte aligned.
3861          */
3862         if (!hba->ucdl_base_addr ||
3863             WARN_ON(hba->ucdl_dma_addr & (128 - 1))) {
3864                 dev_err(hba->dev,
3865                         "Command Descriptor Memory allocation failed\n");
3866                 goto out;
3867         }
3868
3869         /*
3870          * Allocate memory for UTP Transfer descriptors
3871          * UFSHCI requires 1KB alignment of UTRD
3872          */
3873         utrdl_size = (sizeof(struct utp_transfer_req_desc) * hba->nutrs);
3874         hba->utrdl_base_addr = dmam_alloc_coherent(hba->dev,
3875                                                    utrdl_size,
3876                                                    &hba->utrdl_dma_addr,
3877                                                    GFP_KERNEL);
3878         if (!hba->utrdl_base_addr ||
3879             WARN_ON(hba->utrdl_dma_addr & (SZ_1K - 1))) {
3880                 dev_err(hba->dev,
3881                         "Transfer Descriptor Memory allocation failed\n");
3882                 goto out;
3883         }
3884
3885         /*
3886          * Skip utmrdl allocation; it may have been
3887          * allocated during first pass and not released during
3888          * MCQ memory allocation.
3889          * See ufshcd_release_sdb_queue() and ufshcd_config_mcq()
3890          */
3891         if (hba->utmrdl_base_addr)
3892                 goto skip_utmrdl;
3893         /*
3894          * Allocate memory for UTP Task Management descriptors
3895          * UFSHCI requires 1KB alignment of UTMRD
3896          */
3897         utmrdl_size = sizeof(struct utp_task_req_desc) * hba->nutmrs;
3898         hba->utmrdl_base_addr = dmam_alloc_coherent(hba->dev,
3899                                                     utmrdl_size,
3900                                                     &hba->utmrdl_dma_addr,
3901                                                     GFP_KERNEL);
3902         if (!hba->utmrdl_base_addr ||
3903             WARN_ON(hba->utmrdl_dma_addr & (SZ_1K - 1))) {
3904                 dev_err(hba->dev,
3905                 "Task Management Descriptor Memory allocation failed\n");
3906                 goto out;
3907         }
3908
3909 skip_utmrdl:
3910         /* Allocate memory for local reference block */
3911         hba->lrb = devm_kcalloc(hba->dev,
3912                                 hba->nutrs, sizeof(struct ufshcd_lrb),
3913                                 GFP_KERNEL);
3914         if (!hba->lrb) {
3915                 dev_err(hba->dev, "LRB Memory allocation failed\n");
3916                 goto out;
3917         }
3918         return 0;
3919 out:
3920         return -ENOMEM;
3921 }
3922
3923 /**
3924  * ufshcd_host_memory_configure - configure local reference block with
3925  *                              memory offsets
3926  * @hba: per adapter instance
3927  *
3928  * Configure Host memory space
3929  * 1. Update Corresponding UTRD.UCDBA and UTRD.UCDBAU with UCD DMA
3930  * address.
3931  * 2. Update each UTRD with Response UPIU offset, Response UPIU length
3932  * and PRDT offset.
3933  * 3. Save the corresponding addresses of UTRD, UCD.CMD, UCD.RSP and UCD.PRDT
3934  * into local reference block.
3935  */
3936 static void ufshcd_host_memory_configure(struct ufs_hba *hba)
3937 {
3938         struct utp_transfer_req_desc *utrdlp;
3939         dma_addr_t cmd_desc_dma_addr;
3940         dma_addr_t cmd_desc_element_addr;
3941         u16 response_offset;
3942         u16 prdt_offset;
3943         int cmd_desc_size;
3944         int i;
3945
3946         utrdlp = hba->utrdl_base_addr;
3947
3948         response_offset =
3949                 offsetof(struct utp_transfer_cmd_desc, response_upiu);
3950         prdt_offset =
3951                 offsetof(struct utp_transfer_cmd_desc, prd_table);
3952
3953         cmd_desc_size = ufshcd_get_ucd_size(hba);
3954         cmd_desc_dma_addr = hba->ucdl_dma_addr;
3955
3956         for (i = 0; i < hba->nutrs; i++) {
3957                 /* Configure UTRD with command descriptor base address */
3958                 cmd_desc_element_addr =
3959                                 (cmd_desc_dma_addr + (cmd_desc_size * i));
3960                 utrdlp[i].command_desc_base_addr =
3961                                 cpu_to_le64(cmd_desc_element_addr);
3962
3963                 /* Response upiu and prdt offset should be in double words */
3964                 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) {
3965                         utrdlp[i].response_upiu_offset =
3966                                 cpu_to_le16(response_offset);
3967                         utrdlp[i].prd_table_offset =
3968                                 cpu_to_le16(prdt_offset);
3969                         utrdlp[i].response_upiu_length =
3970                                 cpu_to_le16(ALIGNED_UPIU_SIZE);
3971                 } else {
3972                         utrdlp[i].response_upiu_offset =
3973                                 cpu_to_le16(response_offset >> 2);
3974                         utrdlp[i].prd_table_offset =
3975                                 cpu_to_le16(prdt_offset >> 2);
3976                         utrdlp[i].response_upiu_length =
3977                                 cpu_to_le16(ALIGNED_UPIU_SIZE >> 2);
3978                 }
3979
3980                 ufshcd_init_lrb(hba, &hba->lrb[i], i);
3981         }
3982 }
3983
3984 /**
3985  * ufshcd_dme_link_startup - Notify Unipro to perform link startup
3986  * @hba: per adapter instance
3987  *
3988  * UIC_CMD_DME_LINK_STARTUP command must be issued to Unipro layer,
3989  * in order to initialize the Unipro link startup procedure.
3990  * Once the Unipro links are up, the device connected to the controller
3991  * is detected.
3992  *
3993  * Return: 0 on success, non-zero value on failure.
3994  */
3995 static int ufshcd_dme_link_startup(struct ufs_hba *hba)
3996 {
3997         struct uic_command uic_cmd = {
3998                 .command = UIC_CMD_DME_LINK_STARTUP,
3999         };
4000         int ret;
4001
4002         ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
4003         if (ret)
4004                 dev_dbg(hba->dev,
4005                         "dme-link-startup: error code %d\n", ret);
4006         return ret;
4007 }
4008 /**
4009  * ufshcd_dme_reset - UIC command for DME_RESET
4010  * @hba: per adapter instance
4011  *
4012  * DME_RESET command is issued in order to reset UniPro stack.
4013  * This function now deals with cold reset.
4014  *
4015  * Return: 0 on success, non-zero value on failure.
4016  */
4017 static int ufshcd_dme_reset(struct ufs_hba *hba)
4018 {
4019         struct uic_command uic_cmd = {
4020                 .command = UIC_CMD_DME_RESET,
4021         };
4022         int ret;
4023
4024         ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
4025         if (ret)
4026                 dev_err(hba->dev,
4027                         "dme-reset: error code %d\n", ret);
4028
4029         return ret;
4030 }
4031
4032 int ufshcd_dme_configure_adapt(struct ufs_hba *hba,
4033                                int agreed_gear,
4034                                int adapt_val)
4035 {
4036         int ret;
4037
4038         if (agreed_gear < UFS_HS_G4)
4039                 adapt_val = PA_NO_ADAPT;
4040
4041         ret = ufshcd_dme_set(hba,
4042                              UIC_ARG_MIB(PA_TXHSADAPTTYPE),
4043                              adapt_val);
4044         return ret;
4045 }
4046 EXPORT_SYMBOL_GPL(ufshcd_dme_configure_adapt);
4047
4048 /**
4049  * ufshcd_dme_enable - UIC command for DME_ENABLE
4050  * @hba: per adapter instance
4051  *
4052  * DME_ENABLE command is issued in order to enable UniPro stack.
4053  *
4054  * Return: 0 on success, non-zero value on failure.
4055  */
4056 static int ufshcd_dme_enable(struct ufs_hba *hba)
4057 {
4058         struct uic_command uic_cmd = {
4059                 .command = UIC_CMD_DME_ENABLE,
4060         };
4061         int ret;
4062
4063         ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
4064         if (ret)
4065                 dev_err(hba->dev,
4066                         "dme-enable: error code %d\n", ret);
4067
4068         return ret;
4069 }
4070
4071 static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba)
4072 {
4073         #define MIN_DELAY_BEFORE_DME_CMDS_US    1000
4074         unsigned long min_sleep_time_us;
4075
4076         if (!(hba->quirks & UFSHCD_QUIRK_DELAY_BEFORE_DME_CMDS))
4077                 return;
4078
4079         /*
4080          * last_dme_cmd_tstamp will be 0 only for 1st call to
4081          * this function
4082          */
4083         if (unlikely(!ktime_to_us(hba->last_dme_cmd_tstamp))) {
4084                 min_sleep_time_us = MIN_DELAY_BEFORE_DME_CMDS_US;
4085         } else {
4086                 unsigned long delta =
4087                         (unsigned long) ktime_to_us(
4088                                 ktime_sub(ktime_get(),
4089                                 hba->last_dme_cmd_tstamp));
4090
4091                 if (delta < MIN_DELAY_BEFORE_DME_CMDS_US)
4092                         min_sleep_time_us =
4093                                 MIN_DELAY_BEFORE_DME_CMDS_US - delta;
4094                 else
4095                         min_sleep_time_us = 0; /* no more delay required */
4096         }
4097
4098         if (min_sleep_time_us > 0) {
4099                 /* allow sleep for extra 50us if needed */
4100                 usleep_range(min_sleep_time_us, min_sleep_time_us + 50);
4101         }
4102
4103         /* update the last_dme_cmd_tstamp */
4104         hba->last_dme_cmd_tstamp = ktime_get();
4105 }
4106
4107 /**
4108  * ufshcd_dme_set_attr - UIC command for DME_SET, DME_PEER_SET
4109  * @hba: per adapter instance
4110  * @attr_sel: uic command argument1
4111  * @attr_set: attribute set type as uic command argument2
4112  * @mib_val: setting value as uic command argument3
4113  * @peer: indicate whether peer or local
4114  *
4115  * Return: 0 on success, non-zero value on failure.
4116  */
4117 int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel,
4118                         u8 attr_set, u32 mib_val, u8 peer)
4119 {
4120         struct uic_command uic_cmd = {
4121                 .command = peer ? UIC_CMD_DME_PEER_SET : UIC_CMD_DME_SET,
4122                 .argument1 = attr_sel,
4123                 .argument2 = UIC_ARG_ATTR_TYPE(attr_set),
4124                 .argument3 = mib_val,
4125         };
4126         static const char *const action[] = {
4127                 "dme-set",
4128                 "dme-peer-set"
4129         };
4130         const char *set = action[!!peer];
4131         int ret;
4132         int retries = UFS_UIC_COMMAND_RETRIES;
4133
4134         do {
4135                 /* for peer attributes we retry upon failure */
4136                 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
4137                 if (ret)
4138                         dev_dbg(hba->dev, "%s: attr-id 0x%x val 0x%x error code %d\n",
4139                                 set, UIC_GET_ATTR_ID(attr_sel), mib_val, ret);
4140         } while (ret && peer && --retries);
4141
4142         if (ret)
4143                 dev_err(hba->dev, "%s: attr-id 0x%x val 0x%x failed %d retries\n",
4144                         set, UIC_GET_ATTR_ID(attr_sel), mib_val,
4145                         UFS_UIC_COMMAND_RETRIES - retries);
4146
4147         return ret;
4148 }
4149 EXPORT_SYMBOL_GPL(ufshcd_dme_set_attr);
4150
4151 /**
4152  * ufshcd_dme_get_attr - UIC command for DME_GET, DME_PEER_GET
4153  * @hba: per adapter instance
4154  * @attr_sel: uic command argument1
4155  * @mib_val: the value of the attribute as returned by the UIC command
4156  * @peer: indicate whether peer or local
4157  *
4158  * Return: 0 on success, non-zero value on failure.
4159  */
4160 int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel,
4161                         u32 *mib_val, u8 peer)
4162 {
4163         struct uic_command uic_cmd = {
4164                 .command = peer ? UIC_CMD_DME_PEER_GET : UIC_CMD_DME_GET,
4165                 .argument1 = attr_sel,
4166         };
4167         static const char *const action[] = {
4168                 "dme-get",
4169                 "dme-peer-get"
4170         };
4171         const char *get = action[!!peer];
4172         int ret;
4173         int retries = UFS_UIC_COMMAND_RETRIES;
4174         struct ufs_pa_layer_attr orig_pwr_info;
4175         struct ufs_pa_layer_attr temp_pwr_info;
4176         bool pwr_mode_change = false;
4177
4178         if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)) {
4179                 orig_pwr_info = hba->pwr_info;
4180                 temp_pwr_info = orig_pwr_info;
4181
4182                 if (orig_pwr_info.pwr_tx == FAST_MODE ||
4183                     orig_pwr_info.pwr_rx == FAST_MODE) {
4184                         temp_pwr_info.pwr_tx = FASTAUTO_MODE;
4185                         temp_pwr_info.pwr_rx = FASTAUTO_MODE;
4186                         pwr_mode_change = true;
4187                 } else if (orig_pwr_info.pwr_tx == SLOW_MODE ||
4188                     orig_pwr_info.pwr_rx == SLOW_MODE) {
4189                         temp_pwr_info.pwr_tx = SLOWAUTO_MODE;
4190                         temp_pwr_info.pwr_rx = SLOWAUTO_MODE;
4191                         pwr_mode_change = true;
4192                 }
4193                 if (pwr_mode_change) {
4194                         ret = ufshcd_change_power_mode(hba, &temp_pwr_info);
4195                         if (ret)
4196                                 goto out;
4197                 }
4198         }
4199
4200         do {
4201                 /* for peer attributes we retry upon failure */
4202                 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
4203                 if (ret)
4204                         dev_dbg(hba->dev, "%s: attr-id 0x%x error code %d\n",
4205                                 get, UIC_GET_ATTR_ID(attr_sel), ret);
4206         } while (ret && peer && --retries);
4207
4208         if (ret)
4209                 dev_err(hba->dev, "%s: attr-id 0x%x failed %d retries\n",
4210                         get, UIC_GET_ATTR_ID(attr_sel),
4211                         UFS_UIC_COMMAND_RETRIES - retries);
4212
4213         if (mib_val && !ret)
4214                 *mib_val = uic_cmd.argument3;
4215
4216         if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)
4217             && pwr_mode_change)
4218                 ufshcd_change_power_mode(hba, &orig_pwr_info);
4219 out:
4220         return ret;
4221 }
4222 EXPORT_SYMBOL_GPL(ufshcd_dme_get_attr);
4223
4224 /**
4225  * ufshcd_uic_pwr_ctrl - executes UIC commands (which affects the link power
4226  * state) and waits for it to take effect.
4227  *
4228  * @hba: per adapter instance
4229  * @cmd: UIC command to execute
4230  *
4231  * DME operations like DME_SET(PA_PWRMODE), DME_HIBERNATE_ENTER &
4232  * DME_HIBERNATE_EXIT commands take some time to take its effect on both host
4233  * and device UniPro link and hence it's final completion would be indicated by
4234  * dedicated status bits in Interrupt Status register (UPMS, UHES, UHXS) in
4235  * addition to normal UIC command completion Status (UCCS). This function only
4236  * returns after the relevant status bits indicate the completion.
4237  *
4238  * Return: 0 on success, non-zero value on failure.
4239  */
4240 static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd)
4241 {
4242         DECLARE_COMPLETION_ONSTACK(uic_async_done);
4243         unsigned long flags;
4244         u8 status;
4245         int ret;
4246         bool reenable_intr = false;
4247
4248         mutex_lock(&hba->uic_cmd_mutex);
4249         ufshcd_add_delay_before_dme_cmd(hba);
4250
4251         spin_lock_irqsave(hba->host->host_lock, flags);
4252         if (ufshcd_is_link_broken(hba)) {
4253                 ret = -ENOLINK;
4254                 goto out_unlock;
4255         }
4256         hba->uic_async_done = &uic_async_done;
4257         if (ufshcd_readl(hba, REG_INTERRUPT_ENABLE) & UIC_COMMAND_COMPL) {
4258                 ufshcd_disable_intr(hba, UIC_COMMAND_COMPL);
4259                 /*
4260                  * Make sure UIC command completion interrupt is disabled before
4261                  * issuing UIC command.
4262                  */
4263                 ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
4264                 reenable_intr = true;
4265         }
4266         spin_unlock_irqrestore(hba->host->host_lock, flags);
4267         ret = __ufshcd_send_uic_cmd(hba, cmd);
4268         if (ret) {
4269                 dev_err(hba->dev,
4270                         "pwr ctrl cmd 0x%x with mode 0x%x uic error %d\n",
4271                         cmd->command, cmd->argument3, ret);
4272                 goto out;
4273         }
4274
4275         if (!wait_for_completion_timeout(hba->uic_async_done,
4276                                          msecs_to_jiffies(uic_cmd_timeout))) {
4277                 dev_err(hba->dev,
4278                         "pwr ctrl cmd 0x%x with mode 0x%x completion timeout\n",
4279                         cmd->command, cmd->argument3);
4280
4281                 if (!cmd->cmd_active) {
4282                         dev_err(hba->dev, "%s: Power Mode Change operation has been completed, go check UPMCRS\n",
4283                                 __func__);
4284                         goto check_upmcrs;
4285                 }
4286
4287                 ret = -ETIMEDOUT;
4288                 goto out;
4289         }
4290
4291 check_upmcrs:
4292         status = ufshcd_get_upmcrs(hba);
4293         if (status != PWR_LOCAL) {
4294                 dev_err(hba->dev,
4295                         "pwr ctrl cmd 0x%x failed, host upmcrs:0x%x\n",
4296                         cmd->command, status);
4297                 ret = (status != PWR_OK) ? status : -1;
4298         }
4299 out:
4300         if (ret) {
4301                 ufshcd_print_host_state(hba);
4302                 ufshcd_print_pwr_info(hba);
4303                 ufshcd_print_evt_hist(hba);
4304         }
4305
4306         spin_lock_irqsave(hba->host->host_lock, flags);
4307         hba->active_uic_cmd = NULL;
4308         hba->uic_async_done = NULL;
4309         if (reenable_intr)
4310                 ufshcd_enable_intr(hba, UIC_COMMAND_COMPL);
4311         if (ret) {
4312                 ufshcd_set_link_broken(hba);
4313                 ufshcd_schedule_eh_work(hba);
4314         }
4315 out_unlock:
4316         spin_unlock_irqrestore(hba->host->host_lock, flags);
4317         mutex_unlock(&hba->uic_cmd_mutex);
4318
4319         return ret;
4320 }
4321
4322 /**
4323  * ufshcd_send_bsg_uic_cmd - Send UIC commands requested via BSG layer and retrieve the result
4324  * @hba: per adapter instance
4325  * @uic_cmd: UIC command
4326  *
4327  * Return: 0 only if success.
4328  */
4329 int ufshcd_send_bsg_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
4330 {
4331         int ret;
4332
4333         if (hba->quirks & UFSHCD_QUIRK_BROKEN_UIC_CMD)
4334                 return 0;
4335
4336         ufshcd_hold(hba);
4337
4338         if (uic_cmd->argument1 == UIC_ARG_MIB(PA_PWRMODE) &&
4339             uic_cmd->command == UIC_CMD_DME_SET) {
4340                 ret = ufshcd_uic_pwr_ctrl(hba, uic_cmd);
4341                 goto out;
4342         }
4343
4344         mutex_lock(&hba->uic_cmd_mutex);
4345         ufshcd_add_delay_before_dme_cmd(hba);
4346
4347         ret = __ufshcd_send_uic_cmd(hba, uic_cmd);
4348         if (!ret)
4349                 ret = ufshcd_wait_for_uic_cmd(hba, uic_cmd);
4350
4351         mutex_unlock(&hba->uic_cmd_mutex);
4352
4353 out:
4354         ufshcd_release(hba);
4355         return ret;
4356 }
4357
4358 /**
4359  * ufshcd_uic_change_pwr_mode - Perform the UIC power mode chage
4360  *                              using DME_SET primitives.
4361  * @hba: per adapter instance
4362  * @mode: powr mode value
4363  *
4364  * Return: 0 on success, non-zero value on failure.
4365  */
4366 int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode)
4367 {
4368         struct uic_command uic_cmd = {
4369                 .command = UIC_CMD_DME_SET,
4370                 .argument1 = UIC_ARG_MIB(PA_PWRMODE),
4371                 .argument3 = mode,
4372         };
4373         int ret;
4374
4375         if (hba->quirks & UFSHCD_QUIRK_BROKEN_PA_RXHSUNTERMCAP) {
4376                 ret = ufshcd_dme_set(hba,
4377                                 UIC_ARG_MIB_SEL(PA_RXHSUNTERMCAP, 0), 1);
4378                 if (ret) {
4379                         dev_err(hba->dev, "%s: failed to enable PA_RXHSUNTERMCAP ret %d\n",
4380                                                 __func__, ret);
4381                         goto out;
4382                 }
4383         }
4384
4385         ufshcd_hold(hba);
4386         ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
4387         ufshcd_release(hba);
4388
4389 out:
4390         return ret;
4391 }
4392 EXPORT_SYMBOL_GPL(ufshcd_uic_change_pwr_mode);
4393
4394 int ufshcd_link_recovery(struct ufs_hba *hba)
4395 {
4396         int ret;
4397         unsigned long flags;
4398
4399         spin_lock_irqsave(hba->host->host_lock, flags);
4400         hba->ufshcd_state = UFSHCD_STATE_RESET;
4401         ufshcd_set_eh_in_progress(hba);
4402         spin_unlock_irqrestore(hba->host->host_lock, flags);
4403
4404         /* Reset the attached device */
4405         ufshcd_device_reset(hba);
4406
4407         ret = ufshcd_host_reset_and_restore(hba);
4408
4409         spin_lock_irqsave(hba->host->host_lock, flags);
4410         if (ret)
4411                 hba->ufshcd_state = UFSHCD_STATE_ERROR;
4412         ufshcd_clear_eh_in_progress(hba);
4413         spin_unlock_irqrestore(hba->host->host_lock, flags);
4414
4415         if (ret)
4416                 dev_err(hba->dev, "%s: link recovery failed, err %d",
4417                         __func__, ret);
4418
4419         return ret;
4420 }
4421 EXPORT_SYMBOL_GPL(ufshcd_link_recovery);
4422
4423 int ufshcd_uic_hibern8_enter(struct ufs_hba *hba)
4424 {
4425         struct uic_command uic_cmd = {
4426                 .command = UIC_CMD_DME_HIBER_ENTER,
4427         };
4428         ktime_t start = ktime_get();
4429         int ret;
4430
4431         ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER, PRE_CHANGE);
4432
4433         ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
4434         trace_ufshcd_profile_hibern8(dev_name(hba->dev), "enter",
4435                              ktime_to_us(ktime_sub(ktime_get(), start)), ret);
4436
4437         if (ret)
4438                 dev_err(hba->dev, "%s: hibern8 enter failed. ret = %d\n",
4439                         __func__, ret);
4440         else
4441                 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER,
4442                                                                 POST_CHANGE);
4443
4444         return ret;
4445 }
4446 EXPORT_SYMBOL_GPL(ufshcd_uic_hibern8_enter);
4447
4448 int ufshcd_uic_hibern8_exit(struct ufs_hba *hba)
4449 {
4450         struct uic_command uic_cmd = {
4451                 .command = UIC_CMD_DME_HIBER_EXIT,
4452         };
4453         int ret;
4454         ktime_t start = ktime_get();
4455
4456         ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT, PRE_CHANGE);
4457
4458         ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
4459         trace_ufshcd_profile_hibern8(dev_name(hba->dev), "exit",
4460                              ktime_to_us(ktime_sub(ktime_get(), start)), ret);
4461
4462         if (ret) {
4463                 dev_err(hba->dev, "%s: hibern8 exit failed. ret = %d\n",
4464                         __func__, ret);
4465         } else {
4466                 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT,
4467                                                                 POST_CHANGE);
4468                 hba->ufs_stats.last_hibern8_exit_tstamp = local_clock();
4469                 hba->ufs_stats.hibern8_exit_cnt++;
4470         }
4471
4472         return ret;
4473 }
4474 EXPORT_SYMBOL_GPL(ufshcd_uic_hibern8_exit);
4475
4476 static void ufshcd_configure_auto_hibern8(struct ufs_hba *hba)
4477 {
4478         if (!ufshcd_is_auto_hibern8_supported(hba))
4479                 return;
4480
4481         ufshcd_writel(hba, hba->ahit, REG_AUTO_HIBERNATE_IDLE_TIMER);
4482 }
4483
4484 void ufshcd_auto_hibern8_update(struct ufs_hba *hba, u32 ahit)
4485 {
4486         const u32 cur_ahit = READ_ONCE(hba->ahit);
4487
4488         if (!ufshcd_is_auto_hibern8_supported(hba) || cur_ahit == ahit)
4489                 return;
4490
4491         WRITE_ONCE(hba->ahit, ahit);
4492         if (!pm_runtime_suspended(&hba->ufs_device_wlun->sdev_gendev)) {
4493                 ufshcd_rpm_get_sync(hba);
4494                 ufshcd_hold(hba);
4495                 ufshcd_configure_auto_hibern8(hba);
4496                 ufshcd_release(hba);
4497                 ufshcd_rpm_put_sync(hba);
4498         }
4499 }
4500 EXPORT_SYMBOL_GPL(ufshcd_auto_hibern8_update);
4501
4502  /**
4503  * ufshcd_init_pwr_info - setting the POR (power on reset)
4504  * values in hba power info
4505  * @hba: per-adapter instance
4506  */
4507 static void ufshcd_init_pwr_info(struct ufs_hba *hba)
4508 {
4509         hba->pwr_info.gear_rx = UFS_PWM_G1;
4510         hba->pwr_info.gear_tx = UFS_PWM_G1;
4511         hba->pwr_info.lane_rx = UFS_LANE_1;
4512         hba->pwr_info.lane_tx = UFS_LANE_1;
4513         hba->pwr_info.pwr_rx = SLOWAUTO_MODE;
4514         hba->pwr_info.pwr_tx = SLOWAUTO_MODE;
4515         hba->pwr_info.hs_rate = 0;
4516 }
4517
4518 /**
4519  * ufshcd_get_max_pwr_mode - reads the max power mode negotiated with device
4520  * @hba: per-adapter instance
4521  *
4522  * Return: 0 upon success; < 0 upon failure.
4523  */
4524 static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba)
4525 {
4526         struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info;
4527
4528         if (hba->max_pwr_info.is_valid)
4529                 return 0;
4530
4531         if (hba->quirks & UFSHCD_QUIRK_HIBERN_FASTAUTO) {
4532                 pwr_info->pwr_tx = FASTAUTO_MODE;
4533                 pwr_info->pwr_rx = FASTAUTO_MODE;
4534         } else {
4535                 pwr_info->pwr_tx = FAST_MODE;
4536                 pwr_info->pwr_rx = FAST_MODE;
4537         }
4538         pwr_info->hs_rate = PA_HS_MODE_B;
4539
4540         /* Get the connected lane count */
4541         ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDRXDATALANES),
4542                         &pwr_info->lane_rx);
4543         ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4544                         &pwr_info->lane_tx);
4545
4546         if (!pwr_info->lane_rx || !pwr_info->lane_tx) {
4547                 dev_err(hba->dev, "%s: invalid connected lanes value. rx=%d, tx=%d\n",
4548                                 __func__,
4549                                 pwr_info->lane_rx,
4550                                 pwr_info->lane_tx);
4551                 return -EINVAL;
4552         }
4553
4554         if (pwr_info->lane_rx != pwr_info->lane_tx) {
4555                 dev_err(hba->dev, "%s: asymmetric connected lanes. rx=%d, tx=%d\n",
4556                         __func__,
4557                                 pwr_info->lane_rx,
4558                                 pwr_info->lane_tx);
4559                 return -EINVAL;
4560         }
4561
4562         /*
4563          * First, get the maximum gears of HS speed.
4564          * If a zero value, it means there is no HSGEAR capability.
4565          * Then, get the maximum gears of PWM speed.
4566          */
4567         ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), &pwr_info->gear_rx);
4568         if (!pwr_info->gear_rx) {
4569                 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
4570                                 &pwr_info->gear_rx);
4571                 if (!pwr_info->gear_rx) {
4572                         dev_err(hba->dev, "%s: invalid max pwm rx gear read = %d\n",
4573                                 __func__, pwr_info->gear_rx);
4574                         return -EINVAL;
4575                 }
4576                 pwr_info->pwr_rx = SLOW_MODE;
4577         }
4578
4579         ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR),
4580                         &pwr_info->gear_tx);
4581         if (!pwr_info->gear_tx) {
4582                 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
4583                                 &pwr_info->gear_tx);
4584                 if (!pwr_info->gear_tx) {
4585                         dev_err(hba->dev, "%s: invalid max pwm tx gear read = %d\n",
4586                                 __func__, pwr_info->gear_tx);
4587                         return -EINVAL;
4588                 }
4589                 pwr_info->pwr_tx = SLOW_MODE;
4590         }
4591
4592         hba->max_pwr_info.is_valid = true;
4593         return 0;
4594 }
4595
4596 static int ufshcd_change_power_mode(struct ufs_hba *hba,
4597                              struct ufs_pa_layer_attr *pwr_mode)
4598 {
4599         int ret;
4600
4601         /* if already configured to the requested pwr_mode */
4602         if (!hba->force_pmc &&
4603             pwr_mode->gear_rx == hba->pwr_info.gear_rx &&
4604             pwr_mode->gear_tx == hba->pwr_info.gear_tx &&
4605             pwr_mode->lane_rx == hba->pwr_info.lane_rx &&
4606             pwr_mode->lane_tx == hba->pwr_info.lane_tx &&
4607             pwr_mode->pwr_rx == hba->pwr_info.pwr_rx &&
4608             pwr_mode->pwr_tx == hba->pwr_info.pwr_tx &&
4609             pwr_mode->hs_rate == hba->pwr_info.hs_rate) {
4610                 dev_dbg(hba->dev, "%s: power already configured\n", __func__);
4611                 return 0;
4612         }
4613
4614         /*
4615          * Configure attributes for power mode change with below.
4616          * - PA_RXGEAR, PA_ACTIVERXDATALANES, PA_RXTERMINATION,
4617          * - PA_TXGEAR, PA_ACTIVETXDATALANES, PA_TXTERMINATION,
4618          * - PA_HSSERIES
4619          */
4620         ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXGEAR), pwr_mode->gear_rx);
4621         ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVERXDATALANES),
4622                         pwr_mode->lane_rx);
4623         if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
4624                         pwr_mode->pwr_rx == FAST_MODE)
4625                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), true);
4626         else
4627                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), false);
4628
4629         ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXGEAR), pwr_mode->gear_tx);
4630         ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVETXDATALANES),
4631                         pwr_mode->lane_tx);
4632         if (pwr_mode->pwr_tx == FASTAUTO_MODE ||
4633                         pwr_mode->pwr_tx == FAST_MODE)
4634                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), true);
4635         else
4636                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), false);
4637
4638         if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
4639             pwr_mode->pwr_tx == FASTAUTO_MODE ||
4640             pwr_mode->pwr_rx == FAST_MODE ||
4641             pwr_mode->pwr_tx == FAST_MODE)
4642                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HSSERIES),
4643                                                 pwr_mode->hs_rate);
4644
4645         if (!(hba->quirks & UFSHCD_QUIRK_SKIP_DEF_UNIPRO_TIMEOUT_SETTING)) {
4646                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA0),
4647                                 DL_FC0ProtectionTimeOutVal_Default);
4648                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA1),
4649                                 DL_TC0ReplayTimeOutVal_Default);
4650                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA2),
4651                                 DL_AFC0ReqTimeOutVal_Default);
4652                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA3),
4653                                 DL_FC1ProtectionTimeOutVal_Default);
4654                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA4),
4655                                 DL_TC1ReplayTimeOutVal_Default);
4656                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA5),
4657                                 DL_AFC1ReqTimeOutVal_Default);
4658
4659                 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalFC0ProtectionTimeOutVal),
4660                                 DL_FC0ProtectionTimeOutVal_Default);
4661                 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalTC0ReplayTimeOutVal),
4662                                 DL_TC0ReplayTimeOutVal_Default);
4663                 ufshcd_dme_set(hba, UIC_ARG_MIB(DME_LocalAFC0ReqTimeOutVal),
4664                                 DL_AFC0ReqTimeOutVal_Default);
4665         }
4666
4667         ret = ufshcd_uic_change_pwr_mode(hba, pwr_mode->pwr_rx << 4
4668                         | pwr_mode->pwr_tx);
4669
4670         if (ret) {
4671                 dev_err(hba->dev,
4672                         "%s: power mode change failed %d\n", __func__, ret);
4673         } else {
4674                 memcpy(&hba->pwr_info, pwr_mode,
4675                         sizeof(struct ufs_pa_layer_attr));
4676         }
4677
4678         return ret;
4679 }
4680
4681 /**
4682  * ufshcd_config_pwr_mode - configure a new power mode
4683  * @hba: per-adapter instance
4684  * @desired_pwr_mode: desired power configuration
4685  *
4686  * Return: 0 upon success; < 0 upon failure.
4687  */
4688 int ufshcd_config_pwr_mode(struct ufs_hba *hba,
4689                 struct ufs_pa_layer_attr *desired_pwr_mode)
4690 {
4691         struct ufs_pa_layer_attr final_params = { 0 };
4692         int ret;
4693
4694         ret = ufshcd_vops_pwr_change_notify(hba, PRE_CHANGE,
4695                                         desired_pwr_mode, &final_params);
4696
4697         if (ret)
4698                 memcpy(&final_params, desired_pwr_mode, sizeof(final_params));
4699
4700         ret = ufshcd_change_power_mode(hba, &final_params);
4701
4702         if (!ret)
4703                 ufshcd_vops_pwr_change_notify(hba, POST_CHANGE, NULL,
4704                                         &final_params);
4705
4706         return ret;
4707 }
4708 EXPORT_SYMBOL_GPL(ufshcd_config_pwr_mode);
4709
4710 /**
4711  * ufshcd_complete_dev_init() - checks device readiness
4712  * @hba: per-adapter instance
4713  *
4714  * Set fDeviceInit flag and poll until device toggles it.
4715  *
4716  * Return: 0 upon success; < 0 upon failure.
4717  */
4718 static int ufshcd_complete_dev_init(struct ufs_hba *hba)
4719 {
4720         int err;
4721         bool flag_res = true;
4722         ktime_t timeout;
4723
4724         err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
4725                 QUERY_FLAG_IDN_FDEVICEINIT, 0, NULL);
4726         if (err) {
4727                 dev_err(hba->dev,
4728                         "%s: setting fDeviceInit flag failed with error %d\n",
4729                         __func__, err);
4730                 goto out;
4731         }
4732
4733         /* Poll fDeviceInit flag to be cleared */
4734         timeout = ktime_add_ms(ktime_get(), FDEVICEINIT_COMPL_TIMEOUT);
4735         do {
4736                 err = ufshcd_query_flag(hba, UPIU_QUERY_OPCODE_READ_FLAG,
4737                                         QUERY_FLAG_IDN_FDEVICEINIT, 0, &flag_res);
4738                 if (!flag_res)
4739                         break;
4740                 usleep_range(500, 1000);
4741         } while (ktime_before(ktime_get(), timeout));
4742
4743         if (err) {
4744                 dev_err(hba->dev,
4745                                 "%s: reading fDeviceInit flag failed with error %d\n",
4746                                 __func__, err);
4747         } else if (flag_res) {
4748                 dev_err(hba->dev,
4749                                 "%s: fDeviceInit was not cleared by the device\n",
4750                                 __func__);
4751                 err = -EBUSY;
4752         }
4753 out:
4754         return err;
4755 }
4756
4757 /**
4758  * ufshcd_make_hba_operational - Make UFS controller operational
4759  * @hba: per adapter instance
4760  *
4761  * To bring UFS host controller to operational state,
4762  * 1. Enable required interrupts
4763  * 2. Configure interrupt aggregation
4764  * 3. Program UTRL and UTMRL base address
4765  * 4. Configure run-stop-registers
4766  *
4767  * Return: 0 on success, non-zero value on failure.
4768  */
4769 int ufshcd_make_hba_operational(struct ufs_hba *hba)
4770 {
4771         int err = 0;
4772         u32 reg;
4773
4774         /* Enable required interrupts */
4775         ufshcd_enable_intr(hba, UFSHCD_ENABLE_INTRS);
4776
4777         /* Configure interrupt aggregation */
4778         if (ufshcd_is_intr_aggr_allowed(hba))
4779                 ufshcd_config_intr_aggr(hba, hba->nutrs - 1, INT_AGGR_DEF_TO);
4780         else
4781                 ufshcd_disable_intr_aggr(hba);
4782
4783         /* Configure UTRL and UTMRL base address registers */
4784         ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr),
4785                         REG_UTP_TRANSFER_REQ_LIST_BASE_L);
4786         ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr),
4787                         REG_UTP_TRANSFER_REQ_LIST_BASE_H);
4788         ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr),
4789                         REG_UTP_TASK_REQ_LIST_BASE_L);
4790         ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr),
4791                         REG_UTP_TASK_REQ_LIST_BASE_H);
4792
4793         /*
4794          * UCRDY, UTMRLDY and UTRLRDY bits must be 1
4795          */
4796         reg = ufshcd_readl(hba, REG_CONTROLLER_STATUS);
4797         if (!(ufshcd_get_lists_status(reg))) {
4798                 ufshcd_enable_run_stop_reg(hba);
4799         } else {
4800                 dev_err(hba->dev,
4801                         "Host controller not ready to process requests");
4802                 err = -EIO;
4803         }
4804
4805         return err;
4806 }
4807 EXPORT_SYMBOL_GPL(ufshcd_make_hba_operational);
4808
4809 /**
4810  * ufshcd_hba_stop - Send controller to reset state
4811  * @hba: per adapter instance
4812  */
4813 void ufshcd_hba_stop(struct ufs_hba *hba)
4814 {
4815         unsigned long flags;
4816         int err;
4817
4818         /*
4819          * Obtain the host lock to prevent that the controller is disabled
4820          * while the UFS interrupt handler is active on another CPU.
4821          */
4822         spin_lock_irqsave(hba->host->host_lock, flags);
4823         ufshcd_writel(hba, CONTROLLER_DISABLE,  REG_CONTROLLER_ENABLE);
4824         spin_unlock_irqrestore(hba->host->host_lock, flags);
4825
4826         err = ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE,
4827                                         CONTROLLER_ENABLE, CONTROLLER_DISABLE,
4828                                         10, 1);
4829         if (err)
4830                 dev_err(hba->dev, "%s: Controller disable failed\n", __func__);
4831 }
4832 EXPORT_SYMBOL_GPL(ufshcd_hba_stop);
4833
4834 /**
4835  * ufshcd_hba_execute_hce - initialize the controller
4836  * @hba: per adapter instance
4837  *
4838  * The controller resets itself and controller firmware initialization
4839  * sequence kicks off. When controller is ready it will set
4840  * the Host Controller Enable bit to 1.
4841  *
4842  * Return: 0 on success, non-zero value on failure.
4843  */
4844 static int ufshcd_hba_execute_hce(struct ufs_hba *hba)
4845 {
4846         int retry;
4847
4848         for (retry = 3; retry > 0; retry--) {
4849                 if (ufshcd_is_hba_active(hba))
4850                         /* change controller state to "reset state" */
4851                         ufshcd_hba_stop(hba);
4852
4853                 /* UniPro link is disabled at this point */
4854                 ufshcd_set_link_off(hba);
4855
4856                 ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
4857
4858                 /* start controller initialization sequence */
4859                 ufshcd_hba_start(hba);
4860
4861                 /*
4862                  * To initialize a UFS host controller HCE bit must be set to 1.
4863                  * During initialization the HCE bit value changes from 1->0->1.
4864                  * When the host controller completes initialization sequence
4865                  * it sets the value of HCE bit to 1. The same HCE bit is read back
4866                  * to check if the controller has completed initialization sequence.
4867                  * So without this delay the value HCE = 1, set in the previous
4868                  * instruction might be read back.
4869                  * This delay can be changed based on the controller.
4870                  */
4871                 ufshcd_delay_us(hba->vps->hba_enable_delay_us, 100);
4872
4873                 /* wait for the host controller to complete initialization */
4874                 if (!ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE, CONTROLLER_ENABLE,
4875                                               CONTROLLER_ENABLE, 1000, 50))
4876                         break;
4877
4878                 dev_err(hba->dev, "Enabling the controller failed\n");
4879         }
4880
4881         if (!retry)
4882                 return -EIO;
4883
4884         /* enable UIC related interrupts */
4885         ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
4886
4887         ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
4888
4889         return 0;
4890 }
4891
4892 int ufshcd_hba_enable(struct ufs_hba *hba)
4893 {
4894         int ret;
4895
4896         if (hba->quirks & UFSHCI_QUIRK_BROKEN_HCE) {
4897                 ufshcd_set_link_off(hba);
4898                 ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
4899
4900                 /* enable UIC related interrupts */
4901                 ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
4902                 ret = ufshcd_dme_reset(hba);
4903                 if (ret) {
4904                         dev_err(hba->dev, "DME_RESET failed\n");
4905                         return ret;
4906                 }
4907
4908                 ret = ufshcd_dme_enable(hba);
4909                 if (ret) {
4910                         dev_err(hba->dev, "Enabling DME failed\n");
4911                         return ret;
4912                 }
4913
4914                 ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
4915         } else {
4916                 ret = ufshcd_hba_execute_hce(hba);
4917         }
4918
4919         return ret;
4920 }
4921 EXPORT_SYMBOL_GPL(ufshcd_hba_enable);
4922
4923 static int ufshcd_disable_tx_lcc(struct ufs_hba *hba, bool peer)
4924 {
4925         int tx_lanes = 0, i, err = 0;
4926
4927         if (!peer)
4928                 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4929                                &tx_lanes);
4930         else
4931                 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4932                                     &tx_lanes);
4933         for (i = 0; i < tx_lanes; i++) {
4934                 if (!peer)
4935                         err = ufshcd_dme_set(hba,
4936                                 UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
4937                                         UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
4938                                         0);
4939                 else
4940                         err = ufshcd_dme_peer_set(hba,
4941                                 UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
4942                                         UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
4943                                         0);
4944                 if (err) {
4945                         dev_err(hba->dev, "%s: TX LCC Disable failed, peer = %d, lane = %d, err = %d",
4946                                 __func__, peer, i, err);
4947                         break;
4948                 }
4949         }
4950
4951         return err;
4952 }
4953
4954 static inline int ufshcd_disable_device_tx_lcc(struct ufs_hba *hba)
4955 {
4956         return ufshcd_disable_tx_lcc(hba, true);
4957 }
4958
4959 void ufshcd_update_evt_hist(struct ufs_hba *hba, u32 id, u32 val)
4960 {
4961         struct ufs_event_hist *e;
4962
4963         if (id >= UFS_EVT_CNT)
4964                 return;
4965
4966         e = &hba->ufs_stats.event[id];
4967         e->val[e->pos] = val;
4968         e->tstamp[e->pos] = local_clock();
4969         e->cnt += 1;
4970         e->pos = (e->pos + 1) % UFS_EVENT_HIST_LENGTH;
4971
4972         ufshcd_vops_event_notify(hba, id, &val);
4973 }
4974 EXPORT_SYMBOL_GPL(ufshcd_update_evt_hist);
4975
4976 /**
4977  * ufshcd_link_startup - Initialize unipro link startup
4978  * @hba: per adapter instance
4979  *
4980  * Return: 0 for success, non-zero in case of failure.
4981  */
4982 static int ufshcd_link_startup(struct ufs_hba *hba)
4983 {
4984         int ret;
4985         int retries = DME_LINKSTARTUP_RETRIES;
4986         bool link_startup_again = false;
4987
4988         /*
4989          * If UFS device isn't active then we will have to issue link startup
4990          * 2 times to make sure the device state move to active.
4991          */
4992         if (!ufshcd_is_ufs_dev_active(hba))
4993                 link_startup_again = true;
4994
4995 link_startup:
4996         do {
4997                 ufshcd_vops_link_startup_notify(hba, PRE_CHANGE);
4998
4999                 ret = ufshcd_dme_link_startup(hba);
5000
5001                 /* check if device is detected by inter-connect layer */
5002                 if (!ret && !ufshcd_is_device_present(hba)) {
5003                         ufshcd_update_evt_hist(hba,
5004                                                UFS_EVT_LINK_STARTUP_FAIL,
5005                                                0);
5006                         dev_err(hba->dev, "%s: Device not present\n", __func__);
5007                         ret = -ENXIO;
5008                         goto out;
5009                 }
5010
5011                 /*
5012                  * DME link lost indication is only received when link is up,
5013                  * but we can't be sure if the link is up until link startup
5014                  * succeeds. So reset the local Uni-Pro and try again.
5015                  */
5016                 if (ret && retries && ufshcd_hba_enable(hba)) {
5017                         ufshcd_update_evt_hist(hba,
5018                                                UFS_EVT_LINK_STARTUP_FAIL,
5019                                                (u32)ret);
5020                         goto out;
5021                 }
5022         } while (ret && retries--);
5023
5024         if (ret) {
5025                 /* failed to get the link up... retire */
5026                 ufshcd_update_evt_hist(hba,
5027                                        UFS_EVT_LINK_STARTUP_FAIL,
5028                                        (u32)ret);
5029                 goto out;
5030         }
5031
5032         if (link_startup_again) {
5033                 link_startup_again = false;
5034                 retries = DME_LINKSTARTUP_RETRIES;
5035                 goto link_startup;
5036         }
5037
5038         /* Mark that link is up in PWM-G1, 1-lane, SLOW-AUTO mode */
5039         ufshcd_init_pwr_info(hba);
5040         ufshcd_print_pwr_info(hba);
5041
5042         if (hba->quirks & UFSHCD_QUIRK_BROKEN_LCC) {
5043                 ret = ufshcd_disable_device_tx_lcc(hba);
5044                 if (ret)
5045                         goto out;
5046         }
5047
5048         /* Include any host controller configuration via UIC commands */
5049         ret = ufshcd_vops_link_startup_notify(hba, POST_CHANGE);
5050         if (ret)
5051                 goto out;
5052
5053         /* Clear UECPA once due to LINERESET has happened during LINK_STARTUP */
5054         ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER);
5055         ret = ufshcd_make_hba_operational(hba);
5056 out:
5057         if (ret) {
5058                 dev_err(hba->dev, "link startup failed %d\n", ret);
5059                 ufshcd_print_host_state(hba);
5060                 ufshcd_print_pwr_info(hba);
5061                 ufshcd_print_evt_hist(hba);
5062         }
5063         return ret;
5064 }
5065
5066 /**
5067  * ufshcd_verify_dev_init() - Verify device initialization
5068  * @hba: per-adapter instance
5069  *
5070  * Send NOP OUT UPIU and wait for NOP IN response to check whether the
5071  * device Transport Protocol (UTP) layer is ready after a reset.
5072  * If the UTP layer at the device side is not initialized, it may
5073  * not respond with NOP IN UPIU within timeout of %NOP_OUT_TIMEOUT
5074  * and we retry sending NOP OUT for %NOP_OUT_RETRIES iterations.
5075  *
5076  * Return: 0 upon success; < 0 upon failure.
5077  */
5078 static int ufshcd_verify_dev_init(struct ufs_hba *hba)
5079 {
5080         int err = 0;
5081         int retries;
5082
5083         ufshcd_dev_man_lock(hba);
5084
5085         for (retries = NOP_OUT_RETRIES; retries > 0; retries--) {
5086                 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_NOP,
5087                                           hba->nop_out_timeout);
5088
5089                 if (!err || err == -ETIMEDOUT)
5090                         break;
5091
5092                 dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err);
5093         }
5094
5095         ufshcd_dev_man_unlock(hba);
5096
5097         if (err)
5098                 dev_err(hba->dev, "%s: NOP OUT failed %d\n", __func__, err);
5099         return err;
5100 }
5101
5102 /**
5103  * ufshcd_setup_links - associate link b/w device wlun and other luns
5104  * @sdev: pointer to SCSI device
5105  * @hba: pointer to ufs hba
5106  */
5107 static void ufshcd_setup_links(struct ufs_hba *hba, struct scsi_device *sdev)
5108 {
5109         struct device_link *link;
5110
5111         /*
5112          * Device wlun is the supplier & rest of the luns are consumers.
5113          * This ensures that device wlun suspends after all other luns.
5114          */
5115         if (hba->ufs_device_wlun) {
5116                 link = device_link_add(&sdev->sdev_gendev,
5117                                        &hba->ufs_device_wlun->sdev_gendev,
5118                                        DL_FLAG_PM_RUNTIME | DL_FLAG_RPM_ACTIVE);
5119                 if (!link) {
5120                         dev_err(&sdev->sdev_gendev, "Failed establishing link - %s\n",
5121                                 dev_name(&hba->ufs_device_wlun->sdev_gendev));
5122                         return;
5123                 }
5124                 hba->luns_avail--;
5125                 /* Ignore REPORT_LUN wlun probing */
5126                 if (hba->luns_avail == 1) {
5127                         ufshcd_rpm_put(hba);
5128                         return;
5129                 }
5130         } else {
5131                 /*
5132                  * Device wlun is probed. The assumption is that WLUNs are
5133                  * scanned before other LUNs.
5134                  */
5135                 hba->luns_avail--;
5136         }
5137 }
5138
5139 /**
5140  * ufshcd_lu_init - Initialize the relevant parameters of the LU
5141  * @hba: per-adapter instance
5142  * @sdev: pointer to SCSI device
5143  */
5144 static void ufshcd_lu_init(struct ufs_hba *hba, struct scsi_device *sdev)
5145 {
5146         int len = QUERY_DESC_MAX_SIZE;
5147         u8 lun = ufshcd_scsi_to_upiu_lun(sdev->lun);
5148         u8 lun_qdepth = hba->nutrs;
5149         u8 *desc_buf;
5150         int ret;
5151
5152         desc_buf = kzalloc(len, GFP_KERNEL);
5153         if (!desc_buf)
5154                 goto set_qdepth;
5155
5156         ret = ufshcd_read_unit_desc_param(hba, lun, 0, desc_buf, len);
5157         if (ret < 0) {
5158                 if (ret == -EOPNOTSUPP)
5159                         /* If LU doesn't support unit descriptor, its queue depth is set to 1 */
5160                         lun_qdepth = 1;
5161                 kfree(desc_buf);
5162                 goto set_qdepth;
5163         }
5164
5165         if (desc_buf[UNIT_DESC_PARAM_LU_Q_DEPTH]) {
5166                 /*
5167                  * In per-LU queueing architecture, bLUQueueDepth will not be 0, then we will
5168                  * use the smaller between UFSHCI CAP.NUTRS and UFS LU bLUQueueDepth
5169                  */
5170                 lun_qdepth = min_t(int, desc_buf[UNIT_DESC_PARAM_LU_Q_DEPTH], hba->nutrs);
5171         }
5172         /*
5173          * According to UFS device specification, the write protection mode is only supported by
5174          * normal LU, not supported by WLUN.
5175          */
5176         if (hba->dev_info.f_power_on_wp_en && lun < hba->dev_info.max_lu_supported &&
5177             !hba->dev_info.is_lu_power_on_wp &&
5178             desc_buf[UNIT_DESC_PARAM_LU_WR_PROTECT] == UFS_LU_POWER_ON_WP)
5179                 hba->dev_info.is_lu_power_on_wp = true;
5180
5181         /* In case of RPMB LU, check if advanced RPMB mode is enabled */
5182         if (desc_buf[UNIT_DESC_PARAM_UNIT_INDEX] == UFS_UPIU_RPMB_WLUN &&
5183             desc_buf[RPMB_UNIT_DESC_PARAM_REGION_EN] & BIT(4))
5184                 hba->dev_info.b_advanced_rpmb_en = true;
5185
5186
5187         kfree(desc_buf);
5188 set_qdepth:
5189         /*
5190          * For WLUNs that don't support unit descriptor, queue depth is set to 1. For LUs whose
5191          * bLUQueueDepth == 0, the queue depth is set to a maximum value that host can queue.
5192          */
5193         dev_dbg(hba->dev, "Set LU %x queue depth %d\n", lun, lun_qdepth);
5194         scsi_change_queue_depth(sdev, lun_qdepth);
5195 }
5196
5197 /**
5198  * ufshcd_slave_alloc - handle initial SCSI device configurations
5199  * @sdev: pointer to SCSI device
5200  *
5201  * Return: success.
5202  */
5203 static int ufshcd_slave_alloc(struct scsi_device *sdev)
5204 {
5205         struct ufs_hba *hba;
5206
5207         hba = shost_priv(sdev->host);
5208
5209         /* Mode sense(6) is not supported by UFS, so use Mode sense(10) */
5210         sdev->use_10_for_ms = 1;
5211
5212         /* DBD field should be set to 1 in mode sense(10) */
5213         sdev->set_dbd_for_ms = 1;
5214
5215         /* allow SCSI layer to restart the device in case of errors */
5216         sdev->allow_restart = 1;
5217
5218         /* REPORT SUPPORTED OPERATION CODES is not supported */
5219         sdev->no_report_opcodes = 1;
5220
5221         /* WRITE_SAME command is not supported */
5222         sdev->no_write_same = 1;
5223
5224         ufshcd_lu_init(hba, sdev);
5225
5226         ufshcd_setup_links(hba, sdev);
5227
5228         return 0;
5229 }
5230
5231 /**
5232  * ufshcd_change_queue_depth - change queue depth
5233  * @sdev: pointer to SCSI device
5234  * @depth: required depth to set
5235  *
5236  * Change queue depth and make sure the max. limits are not crossed.
5237  *
5238  * Return: new queue depth.
5239  */
5240 static int ufshcd_change_queue_depth(struct scsi_device *sdev, int depth)
5241 {
5242         return scsi_change_queue_depth(sdev, min(depth, sdev->host->can_queue));
5243 }
5244
5245 /**
5246  * ufshcd_device_configure - adjust SCSI device configurations
5247  * @sdev: pointer to SCSI device
5248  * @lim: queue limits
5249  *
5250  * Return: 0 (success).
5251  */
5252 static int ufshcd_device_configure(struct scsi_device *sdev,
5253                 struct queue_limits *lim)
5254 {
5255         struct ufs_hba *hba = shost_priv(sdev->host);
5256         struct request_queue *q = sdev->request_queue;
5257
5258         lim->dma_pad_mask = PRDT_DATA_BYTE_COUNT_PAD - 1;
5259
5260         /*
5261          * Block runtime-pm until all consumers are added.
5262          * Refer ufshcd_setup_links().
5263          */
5264         if (is_device_wlun(sdev))
5265                 pm_runtime_get_noresume(&sdev->sdev_gendev);
5266         else if (ufshcd_is_rpm_autosuspend_allowed(hba))
5267                 sdev->rpm_autosuspend = 1;
5268         /*
5269          * Do not print messages during runtime PM to avoid never-ending cycles
5270          * of messages written back to storage by user space causing runtime
5271          * resume, causing more messages and so on.
5272          */
5273         sdev->silence_suspend = 1;
5274
5275         if (hba->vops && hba->vops->config_scsi_dev)
5276                 hba->vops->config_scsi_dev(sdev);
5277
5278         ufshcd_crypto_register(hba, q);
5279
5280         return 0;
5281 }
5282
5283 /**
5284  * ufshcd_slave_destroy - remove SCSI device configurations
5285  * @sdev: pointer to SCSI device
5286  */
5287 static void ufshcd_slave_destroy(struct scsi_device *sdev)
5288 {
5289         struct ufs_hba *hba;
5290         unsigned long flags;
5291
5292         hba = shost_priv(sdev->host);
5293
5294         /* Drop the reference as it won't be needed anymore */
5295         if (ufshcd_scsi_to_upiu_lun(sdev->lun) == UFS_UPIU_UFS_DEVICE_WLUN) {
5296                 spin_lock_irqsave(hba->host->host_lock, flags);
5297                 hba->ufs_device_wlun = NULL;
5298                 spin_unlock_irqrestore(hba->host->host_lock, flags);
5299         } else if (hba->ufs_device_wlun) {
5300                 struct device *supplier = NULL;
5301
5302                 /* Ensure UFS Device WLUN exists and does not disappear */
5303                 spin_lock_irqsave(hba->host->host_lock, flags);
5304                 if (hba->ufs_device_wlun) {
5305                         supplier = &hba->ufs_device_wlun->sdev_gendev;
5306                         get_device(supplier);
5307                 }
5308                 spin_unlock_irqrestore(hba->host->host_lock, flags);
5309
5310                 if (supplier) {
5311                         /*
5312                          * If a LUN fails to probe (e.g. absent BOOT WLUN), the
5313                          * device will not have been registered but can still
5314                          * have a device link holding a reference to the device.
5315                          */
5316                         device_link_remove(&sdev->sdev_gendev, supplier);
5317                         put_device(supplier);
5318                 }
5319         }
5320 }
5321
5322 /**
5323  * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status
5324  * @lrbp: pointer to local reference block of completed command
5325  * @scsi_status: SCSI command status
5326  *
5327  * Return: value base on SCSI command status.
5328  */
5329 static inline int
5330 ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status)
5331 {
5332         int result = 0;
5333
5334         switch (scsi_status) {
5335         case SAM_STAT_CHECK_CONDITION:
5336                 ufshcd_copy_sense_data(lrbp);
5337                 fallthrough;
5338         case SAM_STAT_GOOD:
5339                 result |= DID_OK << 16 | scsi_status;
5340                 break;
5341         case SAM_STAT_TASK_SET_FULL:
5342         case SAM_STAT_BUSY:
5343         case SAM_STAT_TASK_ABORTED:
5344                 ufshcd_copy_sense_data(lrbp);
5345                 result |= scsi_status;
5346                 break;
5347         default:
5348                 result |= DID_ERROR << 16;
5349                 break;
5350         } /* end of switch */
5351
5352         return result;
5353 }
5354
5355 /**
5356  * ufshcd_transfer_rsp_status - Get overall status of the response
5357  * @hba: per adapter instance
5358  * @lrbp: pointer to local reference block of completed command
5359  * @cqe: pointer to the completion queue entry
5360  *
5361  * Return: result of the command to notify SCSI midlayer.
5362  */
5363 static inline int
5364 ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp,
5365                            struct cq_entry *cqe)
5366 {
5367         int result = 0;
5368         int scsi_status;
5369         enum utp_ocs ocs;
5370         u8 upiu_flags;
5371         u32 resid;
5372
5373         upiu_flags = lrbp->ucd_rsp_ptr->header.flags;
5374         resid = be32_to_cpu(lrbp->ucd_rsp_ptr->sr.residual_transfer_count);
5375         /*
5376          * Test !overflow instead of underflow to support UFS devices that do
5377          * not set either flag.
5378          */
5379         if (resid && !(upiu_flags & UPIU_RSP_FLAG_OVERFLOW))
5380                 scsi_set_resid(lrbp->cmd, resid);
5381
5382         /* overall command status of utrd */
5383         ocs = ufshcd_get_tr_ocs(lrbp, cqe);
5384
5385         if (hba->quirks & UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR) {
5386                 if (lrbp->ucd_rsp_ptr->header.response ||
5387                     lrbp->ucd_rsp_ptr->header.status)
5388                         ocs = OCS_SUCCESS;
5389         }
5390
5391         switch (ocs) {
5392         case OCS_SUCCESS:
5393                 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
5394                 switch (ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr)) {
5395                 case UPIU_TRANSACTION_RESPONSE:
5396                         /*
5397                          * get the result based on SCSI status response
5398                          * to notify the SCSI midlayer of the command status
5399                          */
5400                         scsi_status = lrbp->ucd_rsp_ptr->header.status;
5401                         result = ufshcd_scsi_cmd_status(lrbp, scsi_status);
5402
5403                         /*
5404                          * Currently we are only supporting BKOPs exception
5405                          * events hence we can ignore BKOPs exception event
5406                          * during power management callbacks. BKOPs exception
5407                          * event is not expected to be raised in runtime suspend
5408                          * callback as it allows the urgent bkops.
5409                          * During system suspend, we are anyway forcefully
5410                          * disabling the bkops and if urgent bkops is needed
5411                          * it will be enabled on system resume. Long term
5412                          * solution could be to abort the system suspend if
5413                          * UFS device needs urgent BKOPs.
5414                          */
5415                         if (!hba->pm_op_in_progress &&
5416                             !ufshcd_eh_in_progress(hba) &&
5417                             ufshcd_is_exception_event(lrbp->ucd_rsp_ptr))
5418                                 /* Flushed in suspend */
5419                                 schedule_work(&hba->eeh_work);
5420                         break;
5421                 case UPIU_TRANSACTION_REJECT_UPIU:
5422                         /* TODO: handle Reject UPIU Response */
5423                         result = DID_ERROR << 16;
5424                         dev_err(hba->dev,
5425                                 "Reject UPIU not fully implemented\n");
5426                         break;
5427                 default:
5428                         dev_err(hba->dev,
5429                                 "Unexpected request response code = %x\n",
5430                                 result);
5431                         result = DID_ERROR << 16;
5432                         break;
5433                 }
5434                 break;
5435         case OCS_ABORTED:
5436         case OCS_INVALID_COMMAND_STATUS:
5437                 result |= DID_REQUEUE << 16;
5438                 dev_warn(hba->dev,
5439                                 "OCS %s from controller for tag %d\n",
5440                                 (ocs == OCS_ABORTED ? "aborted" : "invalid"),
5441                                 lrbp->task_tag);
5442                 break;
5443         case OCS_INVALID_CMD_TABLE_ATTR:
5444         case OCS_INVALID_PRDT_ATTR:
5445         case OCS_MISMATCH_DATA_BUF_SIZE:
5446         case OCS_MISMATCH_RESP_UPIU_SIZE:
5447         case OCS_PEER_COMM_FAILURE:
5448         case OCS_FATAL_ERROR:
5449         case OCS_DEVICE_FATAL_ERROR:
5450         case OCS_INVALID_CRYPTO_CONFIG:
5451         case OCS_GENERAL_CRYPTO_ERROR:
5452         default:
5453                 result |= DID_ERROR << 16;
5454                 dev_err(hba->dev,
5455                                 "OCS error from controller = %x for tag %d\n",
5456                                 ocs, lrbp->task_tag);
5457                 ufshcd_print_evt_hist(hba);
5458                 ufshcd_print_host_state(hba);
5459                 break;
5460         } /* end of switch */
5461
5462         if ((host_byte(result) != DID_OK) &&
5463             (host_byte(result) != DID_REQUEUE) && !hba->silence_err_logs)
5464                 ufshcd_print_tr(hba, lrbp->task_tag, true);
5465         return result;
5466 }
5467
5468 static bool ufshcd_is_auto_hibern8_error(struct ufs_hba *hba,
5469                                          u32 intr_mask)
5470 {
5471         if (!ufshcd_is_auto_hibern8_supported(hba) ||
5472             !ufshcd_is_auto_hibern8_enabled(hba))
5473                 return false;
5474
5475         if (!(intr_mask & UFSHCD_UIC_HIBERN8_MASK))
5476                 return false;
5477
5478         if (hba->active_uic_cmd &&
5479             (hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_ENTER ||
5480             hba->active_uic_cmd->command == UIC_CMD_DME_HIBER_EXIT))
5481                 return false;
5482
5483         return true;
5484 }
5485
5486 /**
5487  * ufshcd_uic_cmd_compl - handle completion of uic command
5488  * @hba: per adapter instance
5489  * @intr_status: interrupt status generated by the controller
5490  *
5491  * Return:
5492  *  IRQ_HANDLED - If interrupt is valid
5493  *  IRQ_NONE    - If invalid interrupt
5494  */
5495 static irqreturn_t ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status)
5496 {
5497         irqreturn_t retval = IRQ_NONE;
5498         struct uic_command *cmd;
5499
5500         spin_lock(hba->host->host_lock);
5501         cmd = hba->active_uic_cmd;
5502         if (WARN_ON_ONCE(!cmd))
5503                 goto unlock;
5504
5505         if (ufshcd_is_auto_hibern8_error(hba, intr_status))
5506                 hba->errors |= (UFSHCD_UIC_HIBERN8_MASK & intr_status);
5507
5508         if (intr_status & UIC_COMMAND_COMPL) {
5509                 cmd->argument2 |= ufshcd_get_uic_cmd_result(hba);
5510                 cmd->argument3 = ufshcd_get_dme_attr_val(hba);
5511                 if (!hba->uic_async_done)
5512                         cmd->cmd_active = 0;
5513                 complete(&cmd->done);
5514                 retval = IRQ_HANDLED;
5515         }
5516
5517         if (intr_status & UFSHCD_UIC_PWR_MASK && hba->uic_async_done) {
5518                 cmd->cmd_active = 0;
5519                 complete(hba->uic_async_done);
5520                 retval = IRQ_HANDLED;
5521         }
5522
5523         if (retval == IRQ_HANDLED)
5524                 ufshcd_add_uic_command_trace(hba, cmd, UFS_CMD_COMP);
5525
5526 unlock:
5527         spin_unlock(hba->host->host_lock);
5528
5529         return retval;
5530 }
5531
5532 /* Release the resources allocated for processing a SCSI command. */
5533 void ufshcd_release_scsi_cmd(struct ufs_hba *hba,
5534                              struct ufshcd_lrb *lrbp)
5535 {
5536         struct scsi_cmnd *cmd = lrbp->cmd;
5537
5538         scsi_dma_unmap(cmd);
5539         ufshcd_crypto_clear_prdt(hba, lrbp);
5540         ufshcd_release(hba);
5541         ufshcd_clk_scaling_update_busy(hba);
5542 }
5543
5544 /**
5545  * ufshcd_compl_one_cqe - handle a completion queue entry
5546  * @hba: per adapter instance
5547  * @task_tag: the task tag of the request to be completed
5548  * @cqe: pointer to the completion queue entry
5549  */
5550 void ufshcd_compl_one_cqe(struct ufs_hba *hba, int task_tag,
5551                           struct cq_entry *cqe)
5552 {
5553         struct ufshcd_lrb *lrbp;
5554         struct scsi_cmnd *cmd;
5555         enum utp_ocs ocs;
5556
5557         lrbp = &hba->lrb[task_tag];
5558         lrbp->compl_time_stamp = ktime_get();
5559         lrbp->compl_time_stamp_local_clock = local_clock();
5560         cmd = lrbp->cmd;
5561         if (cmd) {
5562                 if (unlikely(ufshcd_should_inform_monitor(hba, lrbp)))
5563                         ufshcd_update_monitor(hba, lrbp);
5564                 ufshcd_add_command_trace(hba, task_tag, UFS_CMD_COMP);
5565                 cmd->result = ufshcd_transfer_rsp_status(hba, lrbp, cqe);
5566                 ufshcd_release_scsi_cmd(hba, lrbp);
5567                 /* Do not touch lrbp after scsi done */
5568                 scsi_done(cmd);
5569         } else if (hba->dev_cmd.complete) {
5570                 if (cqe) {
5571                         ocs = le32_to_cpu(cqe->status) & MASK_OCS;
5572                         lrbp->utr_descriptor_ptr->header.ocs = ocs;
5573                 }
5574                 complete(hba->dev_cmd.complete);
5575         }
5576 }
5577
5578 /**
5579  * __ufshcd_transfer_req_compl - handle SCSI and query command completion
5580  * @hba: per adapter instance
5581  * @completed_reqs: bitmask that indicates which requests to complete
5582  */
5583 static void __ufshcd_transfer_req_compl(struct ufs_hba *hba,
5584                                         unsigned long completed_reqs)
5585 {
5586         int tag;
5587
5588         for_each_set_bit(tag, &completed_reqs, hba->nutrs)
5589                 ufshcd_compl_one_cqe(hba, tag, NULL);
5590 }
5591
5592 /* Any value that is not an existing queue number is fine for this constant. */
5593 enum {
5594         UFSHCD_POLL_FROM_INTERRUPT_CONTEXT = -1
5595 };
5596
5597 static void ufshcd_clear_polled(struct ufs_hba *hba,
5598                                 unsigned long *completed_reqs)
5599 {
5600         int tag;
5601
5602         for_each_set_bit(tag, completed_reqs, hba->nutrs) {
5603                 struct scsi_cmnd *cmd = hba->lrb[tag].cmd;
5604
5605                 if (!cmd)
5606                         continue;
5607                 if (scsi_cmd_to_rq(cmd)->cmd_flags & REQ_POLLED)
5608                         __clear_bit(tag, completed_reqs);
5609         }
5610 }
5611
5612 /*
5613  * Return: > 0 if one or more commands have been completed or 0 if no
5614  * requests have been completed.
5615  */
5616 static int ufshcd_poll(struct Scsi_Host *shost, unsigned int queue_num)
5617 {
5618         struct ufs_hba *hba = shost_priv(shost);
5619         unsigned long completed_reqs, flags;
5620         u32 tr_doorbell;
5621         struct ufs_hw_queue *hwq;
5622
5623         if (hba->mcq_enabled) {
5624                 hwq = &hba->uhq[queue_num];
5625
5626                 return ufshcd_mcq_poll_cqe_lock(hba, hwq);
5627         }
5628
5629         spin_lock_irqsave(&hba->outstanding_lock, flags);
5630         tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
5631         completed_reqs = ~tr_doorbell & hba->outstanding_reqs;
5632         WARN_ONCE(completed_reqs & ~hba->outstanding_reqs,
5633                   "completed: %#lx; outstanding: %#lx\n", completed_reqs,
5634                   hba->outstanding_reqs);
5635         if (queue_num == UFSHCD_POLL_FROM_INTERRUPT_CONTEXT) {
5636                 /* Do not complete polled requests from interrupt context. */
5637                 ufshcd_clear_polled(hba, &completed_reqs);
5638         }
5639         hba->outstanding_reqs &= ~completed_reqs;
5640         spin_unlock_irqrestore(&hba->outstanding_lock, flags);
5641
5642         if (completed_reqs)
5643                 __ufshcd_transfer_req_compl(hba, completed_reqs);
5644
5645         return completed_reqs != 0;
5646 }
5647
5648 /**
5649  * ufshcd_mcq_compl_pending_transfer - MCQ mode function. It is
5650  * invoked from the error handler context or ufshcd_host_reset_and_restore()
5651  * to complete the pending transfers and free the resources associated with
5652  * the scsi command.
5653  *
5654  * @hba: per adapter instance
5655  * @force_compl: This flag is set to true when invoked
5656  * from ufshcd_host_reset_and_restore() in which case it requires special
5657  * handling because the host controller has been reset by ufshcd_hba_stop().
5658  */
5659 static void ufshcd_mcq_compl_pending_transfer(struct ufs_hba *hba,
5660                                               bool force_compl)
5661 {
5662         struct ufs_hw_queue *hwq;
5663         struct ufshcd_lrb *lrbp;
5664         struct scsi_cmnd *cmd;
5665         unsigned long flags;
5666         int tag;
5667
5668         for (tag = 0; tag < hba->nutrs; tag++) {
5669                 lrbp = &hba->lrb[tag];
5670                 cmd = lrbp->cmd;
5671                 if (!ufshcd_cmd_inflight(cmd) ||
5672                     test_bit(SCMD_STATE_COMPLETE, &cmd->state))
5673                         continue;
5674
5675                 hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(cmd));
5676
5677                 if (force_compl) {
5678                         ufshcd_mcq_compl_all_cqes_lock(hba, hwq);
5679                         /*
5680                          * For those cmds of which the cqes are not present
5681                          * in the cq, complete them explicitly.
5682                          */
5683                         spin_lock_irqsave(&hwq->cq_lock, flags);
5684                         if (cmd && !test_bit(SCMD_STATE_COMPLETE, &cmd->state)) {
5685                                 set_host_byte(cmd, DID_REQUEUE);
5686                                 ufshcd_release_scsi_cmd(hba, lrbp);
5687                                 scsi_done(cmd);
5688                         }
5689                         spin_unlock_irqrestore(&hwq->cq_lock, flags);
5690                 } else {
5691                         ufshcd_mcq_poll_cqe_lock(hba, hwq);
5692                 }
5693         }
5694 }
5695
5696 /**
5697  * ufshcd_transfer_req_compl - handle SCSI and query command completion
5698  * @hba: per adapter instance
5699  *
5700  * Return:
5701  *  IRQ_HANDLED - If interrupt is valid
5702  *  IRQ_NONE    - If invalid interrupt
5703  */
5704 static irqreturn_t ufshcd_transfer_req_compl(struct ufs_hba *hba)
5705 {
5706         /* Resetting interrupt aggregation counters first and reading the
5707          * DOOR_BELL afterward allows us to handle all the completed requests.
5708          * In order to prevent other interrupts starvation the DB is read once
5709          * after reset. The down side of this solution is the possibility of
5710          * false interrupt if device completes another request after resetting
5711          * aggregation and before reading the DB.
5712          */
5713         if (ufshcd_is_intr_aggr_allowed(hba) &&
5714             !(hba->quirks & UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR))
5715                 ufshcd_reset_intr_aggr(hba);
5716
5717         if (ufs_fail_completion(hba))
5718                 return IRQ_HANDLED;
5719
5720         /*
5721          * Ignore the ufshcd_poll() return value and return IRQ_HANDLED since we
5722          * do not want polling to trigger spurious interrupt complaints.
5723          */
5724         ufshcd_poll(hba->host, UFSHCD_POLL_FROM_INTERRUPT_CONTEXT);
5725
5726         return IRQ_HANDLED;
5727 }
5728
5729 int __ufshcd_write_ee_control(struct ufs_hba *hba, u32 ee_ctrl_mask)
5730 {
5731         return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
5732                                        QUERY_ATTR_IDN_EE_CONTROL, 0, 0,
5733                                        &ee_ctrl_mask);
5734 }
5735
5736 int ufshcd_write_ee_control(struct ufs_hba *hba)
5737 {
5738         int err;
5739
5740         mutex_lock(&hba->ee_ctrl_mutex);
5741         err = __ufshcd_write_ee_control(hba, hba->ee_ctrl_mask);
5742         mutex_unlock(&hba->ee_ctrl_mutex);
5743         if (err)
5744                 dev_err(hba->dev, "%s: failed to write ee control %d\n",
5745                         __func__, err);
5746         return err;
5747 }
5748
5749 int ufshcd_update_ee_control(struct ufs_hba *hba, u16 *mask,
5750                              const u16 *other_mask, u16 set, u16 clr)
5751 {
5752         u16 new_mask, ee_ctrl_mask;
5753         int err = 0;
5754
5755         mutex_lock(&hba->ee_ctrl_mutex);
5756         new_mask = (*mask & ~clr) | set;
5757         ee_ctrl_mask = new_mask | *other_mask;
5758         if (ee_ctrl_mask != hba->ee_ctrl_mask)
5759                 err = __ufshcd_write_ee_control(hba, ee_ctrl_mask);
5760         /* Still need to update 'mask' even if 'ee_ctrl_mask' was unchanged */
5761         if (!err) {
5762                 hba->ee_ctrl_mask = ee_ctrl_mask;
5763                 *mask = new_mask;
5764         }
5765         mutex_unlock(&hba->ee_ctrl_mutex);
5766         return err;
5767 }
5768
5769 /**
5770  * ufshcd_disable_ee - disable exception event
5771  * @hba: per-adapter instance
5772  * @mask: exception event to disable
5773  *
5774  * Disables exception event in the device so that the EVENT_ALERT
5775  * bit is not set.
5776  *
5777  * Return: zero on success, non-zero error value on failure.
5778  */
5779 static inline int ufshcd_disable_ee(struct ufs_hba *hba, u16 mask)
5780 {
5781         return ufshcd_update_ee_drv_mask(hba, 0, mask);
5782 }
5783
5784 /**
5785  * ufshcd_enable_ee - enable exception event
5786  * @hba: per-adapter instance
5787  * @mask: exception event to enable
5788  *
5789  * Enable corresponding exception event in the device to allow
5790  * device to alert host in critical scenarios.
5791  *
5792  * Return: zero on success, non-zero error value on failure.
5793  */
5794 static inline int ufshcd_enable_ee(struct ufs_hba *hba, u16 mask)
5795 {
5796         return ufshcd_update_ee_drv_mask(hba, mask, 0);
5797 }
5798
5799 /**
5800  * ufshcd_enable_auto_bkops - Allow device managed BKOPS
5801  * @hba: per-adapter instance
5802  *
5803  * Allow device to manage background operations on its own. Enabling
5804  * this might lead to inconsistent latencies during normal data transfers
5805  * as the device is allowed to manage its own way of handling background
5806  * operations.
5807  *
5808  * Return: zero on success, non-zero on failure.
5809  */
5810 static int ufshcd_enable_auto_bkops(struct ufs_hba *hba)
5811 {
5812         int err = 0;
5813
5814         if (hba->auto_bkops_enabled)
5815                 goto out;
5816
5817         err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
5818                         QUERY_FLAG_IDN_BKOPS_EN, 0, NULL);
5819         if (err) {
5820                 dev_err(hba->dev, "%s: failed to enable bkops %d\n",
5821                                 __func__, err);
5822                 goto out;
5823         }
5824
5825         hba->auto_bkops_enabled = true;
5826         trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Enabled");
5827
5828         /* No need of URGENT_BKOPS exception from the device */
5829         err = ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
5830         if (err)
5831                 dev_err(hba->dev, "%s: failed to disable exception event %d\n",
5832                                 __func__, err);
5833 out:
5834         return err;
5835 }
5836
5837 /**
5838  * ufshcd_disable_auto_bkops - block device in doing background operations
5839  * @hba: per-adapter instance
5840  *
5841  * Disabling background operations improves command response latency but
5842  * has drawback of device moving into critical state where the device is
5843  * not-operable. Make sure to call ufshcd_enable_auto_bkops() whenever the
5844  * host is idle so that BKOPS are managed effectively without any negative
5845  * impacts.
5846  *
5847  * Return: zero on success, non-zero on failure.
5848  */
5849 static int ufshcd_disable_auto_bkops(struct ufs_hba *hba)
5850 {
5851         int err = 0;
5852
5853         if (!hba->auto_bkops_enabled)
5854                 goto out;
5855
5856         /*
5857          * If host assisted BKOPs is to be enabled, make sure
5858          * urgent bkops exception is allowed.
5859          */
5860         err = ufshcd_enable_ee(hba, MASK_EE_URGENT_BKOPS);
5861         if (err) {
5862                 dev_err(hba->dev, "%s: failed to enable exception event %d\n",
5863                                 __func__, err);
5864                 goto out;
5865         }
5866
5867         err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_CLEAR_FLAG,
5868                         QUERY_FLAG_IDN_BKOPS_EN, 0, NULL);
5869         if (err) {
5870                 dev_err(hba->dev, "%s: failed to disable bkops %d\n",
5871                                 __func__, err);
5872                 ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
5873                 goto out;
5874         }
5875
5876         hba->auto_bkops_enabled = false;
5877         trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Disabled");
5878         hba->is_urgent_bkops_lvl_checked = false;
5879 out:
5880         return err;
5881 }
5882
5883 /**
5884  * ufshcd_force_reset_auto_bkops - force reset auto bkops state
5885  * @hba: per adapter instance
5886  *
5887  * After a device reset the device may toggle the BKOPS_EN flag
5888  * to default value. The s/w tracking variables should be updated
5889  * as well. This function would change the auto-bkops state based on
5890  * UFSHCD_CAP_KEEP_AUTO_BKOPS_ENABLED_EXCEPT_SUSPEND.
5891  */
5892 static void ufshcd_force_reset_auto_bkops(struct ufs_hba *hba)
5893 {
5894         if (ufshcd_keep_autobkops_enabled_except_suspend(hba)) {
5895                 hba->auto_bkops_enabled = false;
5896                 hba->ee_ctrl_mask |= MASK_EE_URGENT_BKOPS;
5897                 ufshcd_enable_auto_bkops(hba);
5898         } else {
5899                 hba->auto_bkops_enabled = true;
5900                 hba->ee_ctrl_mask &= ~MASK_EE_URGENT_BKOPS;
5901                 ufshcd_disable_auto_bkops(hba);
5902         }
5903         hba->urgent_bkops_lvl = BKOPS_STATUS_PERF_IMPACT;
5904         hba->is_urgent_bkops_lvl_checked = false;
5905 }
5906
5907 static inline int ufshcd_get_bkops_status(struct ufs_hba *hba, u32 *status)
5908 {
5909         return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5910                         QUERY_ATTR_IDN_BKOPS_STATUS, 0, 0, status);
5911 }
5912
5913 /**
5914  * ufshcd_bkops_ctrl - control the auto bkops based on current bkops status
5915  * @hba: per-adapter instance
5916  *
5917  * Read the bkops_status from the UFS device and Enable fBackgroundOpsEn
5918  * flag in the device to permit background operations if the device
5919  * bkops_status is greater than or equal to the "hba->urgent_bkops_lvl",
5920  * disable otherwise.
5921  *
5922  * Return: 0 for success, non-zero in case of failure.
5923  *
5924  * NOTE: Caller of this function can check the "hba->auto_bkops_enabled" flag
5925  * to know whether auto bkops is enabled or disabled after this function
5926  * returns control to it.
5927  */
5928 static int ufshcd_bkops_ctrl(struct ufs_hba *hba)
5929 {
5930         enum bkops_status status = hba->urgent_bkops_lvl;
5931         u32 curr_status = 0;
5932         int err;
5933
5934         err = ufshcd_get_bkops_status(hba, &curr_status);
5935         if (err) {
5936                 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
5937                                 __func__, err);
5938                 goto out;
5939         } else if (curr_status > BKOPS_STATUS_MAX) {
5940                 dev_err(hba->dev, "%s: invalid BKOPS status %d\n",
5941                                 __func__, curr_status);
5942                 err = -EINVAL;
5943                 goto out;
5944         }
5945
5946         if (curr_status >= status)
5947                 err = ufshcd_enable_auto_bkops(hba);
5948         else
5949                 err = ufshcd_disable_auto_bkops(hba);
5950 out:
5951         return err;
5952 }
5953
5954 static inline int ufshcd_get_ee_status(struct ufs_hba *hba, u32 *status)
5955 {
5956         return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5957                         QUERY_ATTR_IDN_EE_STATUS, 0, 0, status);
5958 }
5959
5960 static void ufshcd_bkops_exception_event_handler(struct ufs_hba *hba)
5961 {
5962         int err;
5963         u32 curr_status = 0;
5964
5965         if (hba->is_urgent_bkops_lvl_checked)
5966                 goto enable_auto_bkops;
5967
5968         err = ufshcd_get_bkops_status(hba, &curr_status);
5969         if (err) {
5970                 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
5971                                 __func__, err);
5972                 goto out;
5973         }
5974
5975         /*
5976          * We are seeing that some devices are raising the urgent bkops
5977          * exception events even when BKOPS status doesn't indicate performace
5978          * impacted or critical. Handle these device by determining their urgent
5979          * bkops status at runtime.
5980          */
5981         if (curr_status < BKOPS_STATUS_PERF_IMPACT) {
5982                 dev_err(hba->dev, "%s: device raised urgent BKOPS exception for bkops status %d\n",
5983                                 __func__, curr_status);
5984                 /* update the current status as the urgent bkops level */
5985                 hba->urgent_bkops_lvl = curr_status;
5986                 hba->is_urgent_bkops_lvl_checked = true;
5987         }
5988
5989 enable_auto_bkops:
5990         err = ufshcd_enable_auto_bkops(hba);
5991 out:
5992         if (err < 0)
5993                 dev_err(hba->dev, "%s: failed to handle urgent bkops %d\n",
5994                                 __func__, err);
5995 }
5996
5997 static void ufshcd_temp_exception_event_handler(struct ufs_hba *hba, u16 status)
5998 {
5999         u32 value;
6000
6001         if (ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
6002                                 QUERY_ATTR_IDN_CASE_ROUGH_TEMP, 0, 0, &value))
6003                 return;
6004
6005         dev_info(hba->dev, "exception Tcase %d\n", value - 80);
6006
6007         ufs_hwmon_notify_event(hba, status & MASK_EE_URGENT_TEMP);
6008
6009         /*
6010          * A placeholder for the platform vendors to add whatever additional
6011          * steps required
6012          */
6013 }
6014
6015 static int __ufshcd_wb_toggle(struct ufs_hba *hba, bool set, enum flag_idn idn)
6016 {
6017         u8 index;
6018         enum query_opcode opcode = set ? UPIU_QUERY_OPCODE_SET_FLAG :
6019                                    UPIU_QUERY_OPCODE_CLEAR_FLAG;
6020
6021         index = ufshcd_wb_get_query_index(hba);
6022         return ufshcd_query_flag_retry(hba, opcode, idn, index, NULL);
6023 }
6024
6025 int ufshcd_wb_toggle(struct ufs_hba *hba, bool enable)
6026 {
6027         int ret;
6028
6029         if (!ufshcd_is_wb_allowed(hba) ||
6030             hba->dev_info.wb_enabled == enable)
6031                 return 0;
6032
6033         ret = __ufshcd_wb_toggle(hba, enable, QUERY_FLAG_IDN_WB_EN);
6034         if (ret) {
6035                 dev_err(hba->dev, "%s: Write Booster %s failed %d\n",
6036                         __func__, enable ? "enabling" : "disabling", ret);
6037                 return ret;
6038         }
6039
6040         hba->dev_info.wb_enabled = enable;
6041         dev_dbg(hba->dev, "%s: Write Booster %s\n",
6042                         __func__, enable ? "enabled" : "disabled");
6043
6044         return ret;
6045 }
6046
6047 static void ufshcd_wb_toggle_buf_flush_during_h8(struct ufs_hba *hba,
6048                                                  bool enable)
6049 {
6050         int ret;
6051
6052         ret = __ufshcd_wb_toggle(hba, enable,
6053                         QUERY_FLAG_IDN_WB_BUFF_FLUSH_DURING_HIBERN8);
6054         if (ret) {
6055                 dev_err(hba->dev, "%s: WB-Buf Flush during H8 %s failed %d\n",
6056                         __func__, enable ? "enabling" : "disabling", ret);
6057                 return;
6058         }
6059         dev_dbg(hba->dev, "%s: WB-Buf Flush during H8 %s\n",
6060                         __func__, enable ? "enabled" : "disabled");
6061 }
6062
6063 int ufshcd_wb_toggle_buf_flush(struct ufs_hba *hba, bool enable)
6064 {
6065         int ret;
6066
6067         if (!ufshcd_is_wb_allowed(hba) ||
6068             hba->dev_info.wb_buf_flush_enabled == enable)
6069                 return 0;
6070
6071         ret = __ufshcd_wb_toggle(hba, enable, QUERY_FLAG_IDN_WB_BUFF_FLUSH_EN);
6072         if (ret) {
6073                 dev_err(hba->dev, "%s: WB-Buf Flush %s failed %d\n",
6074                         __func__, enable ? "enabling" : "disabling", ret);
6075                 return ret;
6076         }
6077
6078         hba->dev_info.wb_buf_flush_enabled = enable;
6079         dev_dbg(hba->dev, "%s: WB-Buf Flush %s\n",
6080                         __func__, enable ? "enabled" : "disabled");
6081
6082         return ret;
6083 }
6084
6085 static bool ufshcd_wb_presrv_usrspc_keep_vcc_on(struct ufs_hba *hba,
6086                                                 u32 avail_buf)
6087 {
6088         u32 cur_buf;
6089         int ret;
6090         u8 index;
6091
6092         index = ufshcd_wb_get_query_index(hba);
6093         ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
6094                                               QUERY_ATTR_IDN_CURR_WB_BUFF_SIZE,
6095                                               index, 0, &cur_buf);
6096         if (ret) {
6097                 dev_err(hba->dev, "%s: dCurWriteBoosterBufferSize read failed %d\n",
6098                         __func__, ret);
6099                 return false;
6100         }
6101
6102         if (!cur_buf) {
6103                 dev_info(hba->dev, "dCurWBBuf: %d WB disabled until free-space is available\n",
6104                          cur_buf);
6105                 return false;
6106         }
6107         /* Let it continue to flush when available buffer exceeds threshold */
6108         return avail_buf < hba->vps->wb_flush_threshold;
6109 }
6110
6111 static void ufshcd_wb_force_disable(struct ufs_hba *hba)
6112 {
6113         if (ufshcd_is_wb_buf_flush_allowed(hba))
6114                 ufshcd_wb_toggle_buf_flush(hba, false);
6115
6116         ufshcd_wb_toggle_buf_flush_during_h8(hba, false);
6117         ufshcd_wb_toggle(hba, false);
6118         hba->caps &= ~UFSHCD_CAP_WB_EN;
6119
6120         dev_info(hba->dev, "%s: WB force disabled\n", __func__);
6121 }
6122
6123 static bool ufshcd_is_wb_buf_lifetime_available(struct ufs_hba *hba)
6124 {
6125         u32 lifetime;
6126         int ret;
6127         u8 index;
6128
6129         index = ufshcd_wb_get_query_index(hba);
6130         ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
6131                                       QUERY_ATTR_IDN_WB_BUFF_LIFE_TIME_EST,
6132                                       index, 0, &lifetime);
6133         if (ret) {
6134                 dev_err(hba->dev,
6135                         "%s: bWriteBoosterBufferLifeTimeEst read failed %d\n",
6136                         __func__, ret);
6137                 return false;
6138         }
6139
6140         if (lifetime == UFS_WB_EXCEED_LIFETIME) {
6141                 dev_err(hba->dev, "%s: WB buf lifetime is exhausted 0x%02X\n",
6142                         __func__, lifetime);
6143                 return false;
6144         }
6145
6146         dev_dbg(hba->dev, "%s: WB buf lifetime is 0x%02X\n",
6147                 __func__, lifetime);
6148
6149         return true;
6150 }
6151
6152 static bool ufshcd_wb_need_flush(struct ufs_hba *hba)
6153 {
6154         int ret;
6155         u32 avail_buf;
6156         u8 index;
6157
6158         if (!ufshcd_is_wb_allowed(hba))
6159                 return false;
6160
6161         if (!ufshcd_is_wb_buf_lifetime_available(hba)) {
6162                 ufshcd_wb_force_disable(hba);
6163                 return false;
6164         }
6165
6166         /*
6167          * The ufs device needs the vcc to be ON to flush.
6168          * With user-space reduction enabled, it's enough to enable flush
6169          * by checking only the available buffer. The threshold
6170          * defined here is > 90% full.
6171          * With user-space preserved enabled, the current-buffer
6172          * should be checked too because the wb buffer size can reduce
6173          * when disk tends to be full. This info is provided by current
6174          * buffer (dCurrentWriteBoosterBufferSize). There's no point in
6175          * keeping vcc on when current buffer is empty.
6176          */
6177         index = ufshcd_wb_get_query_index(hba);
6178         ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
6179                                       QUERY_ATTR_IDN_AVAIL_WB_BUFF_SIZE,
6180                                       index, 0, &avail_buf);
6181         if (ret) {
6182                 dev_warn(hba->dev, "%s: dAvailableWriteBoosterBufferSize read failed %d\n",
6183                          __func__, ret);
6184                 return false;
6185         }
6186
6187         if (!hba->dev_info.b_presrv_uspc_en)
6188                 return avail_buf <= UFS_WB_BUF_REMAIN_PERCENT(10);
6189
6190         return ufshcd_wb_presrv_usrspc_keep_vcc_on(hba, avail_buf);
6191 }
6192
6193 static void ufshcd_rpm_dev_flush_recheck_work(struct work_struct *work)
6194 {
6195         struct ufs_hba *hba = container_of(to_delayed_work(work),
6196                                            struct ufs_hba,
6197                                            rpm_dev_flush_recheck_work);
6198         /*
6199          * To prevent unnecessary VCC power drain after device finishes
6200          * WriteBooster buffer flush or Auto BKOPs, force runtime resume
6201          * after a certain delay to recheck the threshold by next runtime
6202          * suspend.
6203          */
6204         ufshcd_rpm_get_sync(hba);
6205         ufshcd_rpm_put_sync(hba);
6206 }
6207
6208 /**
6209  * ufshcd_exception_event_handler - handle exceptions raised by device
6210  * @work: pointer to work data
6211  *
6212  * Read bExceptionEventStatus attribute from the device and handle the
6213  * exception event accordingly.
6214  */
6215 static void ufshcd_exception_event_handler(struct work_struct *work)
6216 {
6217         struct ufs_hba *hba;
6218         int err;
6219         u32 status = 0;
6220         hba = container_of(work, struct ufs_hba, eeh_work);
6221
6222         err = ufshcd_get_ee_status(hba, &status);
6223         if (err) {
6224                 dev_err(hba->dev, "%s: failed to get exception status %d\n",
6225                                 __func__, err);
6226                 return;
6227         }
6228
6229         trace_ufshcd_exception_event(dev_name(hba->dev), status);
6230
6231         if (status & hba->ee_drv_mask & MASK_EE_URGENT_BKOPS)
6232                 ufshcd_bkops_exception_event_handler(hba);
6233
6234         if (status & hba->ee_drv_mask & MASK_EE_URGENT_TEMP)
6235                 ufshcd_temp_exception_event_handler(hba, status);
6236
6237         ufs_debugfs_exception_event(hba, status);
6238 }
6239
6240 /* Complete requests that have door-bell cleared */
6241 static void ufshcd_complete_requests(struct ufs_hba *hba, bool force_compl)
6242 {
6243         if (hba->mcq_enabled)
6244                 ufshcd_mcq_compl_pending_transfer(hba, force_compl);
6245         else
6246                 ufshcd_transfer_req_compl(hba);
6247
6248         ufshcd_tmc_handler(hba);
6249 }
6250
6251 /**
6252  * ufshcd_quirk_dl_nac_errors - This function checks if error handling is
6253  *                              to recover from the DL NAC errors or not.
6254  * @hba: per-adapter instance
6255  *
6256  * Return: true if error handling is required, false otherwise.
6257  */
6258 static bool ufshcd_quirk_dl_nac_errors(struct ufs_hba *hba)
6259 {
6260         unsigned long flags;
6261         bool err_handling = true;
6262
6263         spin_lock_irqsave(hba->host->host_lock, flags);
6264         /*
6265          * UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS only workaround the
6266          * device fatal error and/or DL NAC & REPLAY timeout errors.
6267          */
6268         if (hba->saved_err & (CONTROLLER_FATAL_ERROR | SYSTEM_BUS_FATAL_ERROR))
6269                 goto out;
6270
6271         if ((hba->saved_err & DEVICE_FATAL_ERROR) ||
6272             ((hba->saved_err & UIC_ERROR) &&
6273              (hba->saved_uic_err & UFSHCD_UIC_DL_TCx_REPLAY_ERROR)))
6274                 goto out;
6275
6276         if ((hba->saved_err & UIC_ERROR) &&
6277             (hba->saved_uic_err & UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)) {
6278                 int err;
6279                 /*
6280                  * wait for 50ms to see if we can get any other errors or not.
6281                  */
6282                 spin_unlock_irqrestore(hba->host->host_lock, flags);
6283                 msleep(50);
6284                 spin_lock_irqsave(hba->host->host_lock, flags);
6285
6286                 /*
6287                  * now check if we have got any other severe errors other than
6288                  * DL NAC error?
6289                  */
6290                 if ((hba->saved_err & INT_FATAL_ERRORS) ||
6291                     ((hba->saved_err & UIC_ERROR) &&
6292                     (hba->saved_uic_err & ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)))
6293                         goto out;
6294
6295                 /*
6296                  * As DL NAC is the only error received so far, send out NOP
6297                  * command to confirm if link is still active or not.
6298                  *   - If we don't get any response then do error recovery.
6299                  *   - If we get response then clear the DL NAC error bit.
6300                  */
6301
6302                 spin_unlock_irqrestore(hba->host->host_lock, flags);
6303                 err = ufshcd_verify_dev_init(hba);
6304                 spin_lock_irqsave(hba->host->host_lock, flags);
6305
6306                 if (err)
6307                         goto out;
6308
6309                 /* Link seems to be alive hence ignore the DL NAC errors */
6310                 if (hba->saved_uic_err == UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)
6311                         hba->saved_err &= ~UIC_ERROR;
6312                 /* clear NAC error */
6313                 hba->saved_uic_err &= ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
6314                 if (!hba->saved_uic_err)
6315                         err_handling = false;
6316         }
6317 out:
6318         spin_unlock_irqrestore(hba->host->host_lock, flags);
6319         return err_handling;
6320 }
6321
6322 /* host lock must be held before calling this func */
6323 static inline bool ufshcd_is_saved_err_fatal(struct ufs_hba *hba)
6324 {
6325         return (hba->saved_uic_err & UFSHCD_UIC_DL_PA_INIT_ERROR) ||
6326                (hba->saved_err & (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK));
6327 }
6328
6329 void ufshcd_schedule_eh_work(struct ufs_hba *hba)
6330 {
6331         lockdep_assert_held(hba->host->host_lock);
6332
6333         /* handle fatal errors only when link is not in error state */
6334         if (hba->ufshcd_state != UFSHCD_STATE_ERROR) {
6335                 if (hba->force_reset || ufshcd_is_link_broken(hba) ||
6336                     ufshcd_is_saved_err_fatal(hba))
6337                         hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED_FATAL;
6338                 else
6339                         hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED_NON_FATAL;
6340                 queue_work(hba->eh_wq, &hba->eh_work);
6341         }
6342 }
6343
6344 static void ufshcd_force_error_recovery(struct ufs_hba *hba)
6345 {
6346         spin_lock_irq(hba->host->host_lock);
6347         hba->force_reset = true;
6348         ufshcd_schedule_eh_work(hba);
6349         spin_unlock_irq(hba->host->host_lock);
6350 }
6351
6352 static void ufshcd_clk_scaling_allow(struct ufs_hba *hba, bool allow)
6353 {
6354         mutex_lock(&hba->wb_mutex);
6355         down_write(&hba->clk_scaling_lock);
6356         hba->clk_scaling.is_allowed = allow;
6357         up_write(&hba->clk_scaling_lock);
6358         mutex_unlock(&hba->wb_mutex);
6359 }
6360
6361 static void ufshcd_clk_scaling_suspend(struct ufs_hba *hba, bool suspend)
6362 {
6363         if (suspend) {
6364                 if (hba->clk_scaling.is_enabled)
6365                         ufshcd_suspend_clkscaling(hba);
6366                 ufshcd_clk_scaling_allow(hba, false);
6367         } else {
6368                 ufshcd_clk_scaling_allow(hba, true);
6369                 if (hba->clk_scaling.is_enabled)
6370                         ufshcd_resume_clkscaling(hba);
6371         }
6372 }
6373
6374 static void ufshcd_err_handling_prepare(struct ufs_hba *hba)
6375 {
6376         ufshcd_rpm_get_sync(hba);
6377         if (pm_runtime_status_suspended(&hba->ufs_device_wlun->sdev_gendev) ||
6378             hba->is_sys_suspended) {
6379                 enum ufs_pm_op pm_op;
6380
6381                 /*
6382                  * Don't assume anything of resume, if
6383                  * resume fails, irq and clocks can be OFF, and powers
6384                  * can be OFF or in LPM.
6385                  */
6386                 ufshcd_setup_hba_vreg(hba, true);
6387                 ufshcd_enable_irq(hba);
6388                 ufshcd_setup_vreg(hba, true);
6389                 ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq);
6390                 ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2);
6391                 ufshcd_hold(hba);
6392                 if (!ufshcd_is_clkgating_allowed(hba))
6393                         ufshcd_setup_clocks(hba, true);
6394                 pm_op = hba->is_sys_suspended ? UFS_SYSTEM_PM : UFS_RUNTIME_PM;
6395                 ufshcd_vops_resume(hba, pm_op);
6396         } else {
6397                 ufshcd_hold(hba);
6398                 if (ufshcd_is_clkscaling_supported(hba) &&
6399                     hba->clk_scaling.is_enabled)
6400                         ufshcd_suspend_clkscaling(hba);
6401                 ufshcd_clk_scaling_allow(hba, false);
6402         }
6403         /* Wait for ongoing ufshcd_queuecommand() calls to finish. */
6404         blk_mq_quiesce_tagset(&hba->host->tag_set);
6405         cancel_work_sync(&hba->eeh_work);
6406 }
6407
6408 static void ufshcd_err_handling_unprepare(struct ufs_hba *hba)
6409 {
6410         blk_mq_unquiesce_tagset(&hba->host->tag_set);
6411         ufshcd_release(hba);
6412         if (ufshcd_is_clkscaling_supported(hba))
6413                 ufshcd_clk_scaling_suspend(hba, false);
6414         ufshcd_rpm_put(hba);
6415 }
6416
6417 static inline bool ufshcd_err_handling_should_stop(struct ufs_hba *hba)
6418 {
6419         return (!hba->is_powered || hba->shutting_down ||
6420                 !hba->ufs_device_wlun ||
6421                 hba->ufshcd_state == UFSHCD_STATE_ERROR ||
6422                 (!(hba->saved_err || hba->saved_uic_err || hba->force_reset ||
6423                    ufshcd_is_link_broken(hba))));
6424 }
6425
6426 #ifdef CONFIG_PM
6427 static void ufshcd_recover_pm_error(struct ufs_hba *hba)
6428 {
6429         struct Scsi_Host *shost = hba->host;
6430         struct scsi_device *sdev;
6431         struct request_queue *q;
6432         int ret;
6433
6434         hba->is_sys_suspended = false;
6435         /*
6436          * Set RPM status of wlun device to RPM_ACTIVE,
6437          * this also clears its runtime error.
6438          */
6439         ret = pm_runtime_set_active(&hba->ufs_device_wlun->sdev_gendev);
6440
6441         /* hba device might have a runtime error otherwise */
6442         if (ret)
6443                 ret = pm_runtime_set_active(hba->dev);
6444         /*
6445          * If wlun device had runtime error, we also need to resume those
6446          * consumer scsi devices in case any of them has failed to be
6447          * resumed due to supplier runtime resume failure. This is to unblock
6448          * blk_queue_enter in case there are bios waiting inside it.
6449          */
6450         if (!ret) {
6451                 shost_for_each_device(sdev, shost) {
6452                         q = sdev->request_queue;
6453                         if (q->dev && (q->rpm_status == RPM_SUSPENDED ||
6454                                        q->rpm_status == RPM_SUSPENDING))
6455                                 pm_request_resume(q->dev);
6456                 }
6457         }
6458 }
6459 #else
6460 static inline void ufshcd_recover_pm_error(struct ufs_hba *hba)
6461 {
6462 }
6463 #endif
6464
6465 static bool ufshcd_is_pwr_mode_restore_needed(struct ufs_hba *hba)
6466 {
6467         struct ufs_pa_layer_attr *pwr_info = &hba->pwr_info;
6468         u32 mode;
6469
6470         ufshcd_dme_get(hba, UIC_ARG_MIB(PA_PWRMODE), &mode);
6471
6472         if (pwr_info->pwr_rx != ((mode >> PWRMODE_RX_OFFSET) & PWRMODE_MASK))
6473                 return true;
6474
6475         if (pwr_info->pwr_tx != (mode & PWRMODE_MASK))
6476                 return true;
6477
6478         return false;
6479 }
6480
6481 static bool ufshcd_abort_one(struct request *rq, void *priv)
6482 {
6483         int *ret = priv;
6484         u32 tag = rq->tag;
6485         struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);
6486         struct scsi_device *sdev = cmd->device;
6487         struct Scsi_Host *shost = sdev->host;
6488         struct ufs_hba *hba = shost_priv(shost);
6489
6490         *ret = ufshcd_try_to_abort_task(hba, tag);
6491         dev_err(hba->dev, "Aborting tag %d / CDB %#02x %s\n", tag,
6492                 hba->lrb[tag].cmd ? hba->lrb[tag].cmd->cmnd[0] : -1,
6493                 *ret ? "failed" : "succeeded");
6494
6495         return *ret == 0;
6496 }
6497
6498 /**
6499  * ufshcd_abort_all - Abort all pending commands.
6500  * @hba: Host bus adapter pointer.
6501  *
6502  * Return: true if and only if the host controller needs to be reset.
6503  */
6504 static bool ufshcd_abort_all(struct ufs_hba *hba)
6505 {
6506         int tag, ret = 0;
6507
6508         blk_mq_tagset_busy_iter(&hba->host->tag_set, ufshcd_abort_one, &ret);
6509         if (ret)
6510                 goto out;
6511
6512         /* Clear pending task management requests */
6513         for_each_set_bit(tag, &hba->outstanding_tasks, hba->nutmrs) {
6514                 ret = ufshcd_clear_tm_cmd(hba, tag);
6515                 if (ret)
6516                         goto out;
6517         }
6518
6519 out:
6520         /* Complete the requests that are cleared by s/w */
6521         ufshcd_complete_requests(hba, false);
6522
6523         return ret != 0;
6524 }
6525
6526 /**
6527  * ufshcd_err_handler - handle UFS errors that require s/w attention
6528  * @work: pointer to work structure
6529  */
6530 static void ufshcd_err_handler(struct work_struct *work)
6531 {
6532         int retries = MAX_ERR_HANDLER_RETRIES;
6533         struct ufs_hba *hba;
6534         unsigned long flags;
6535         bool needs_restore;
6536         bool needs_reset;
6537         int pmc_err;
6538
6539         hba = container_of(work, struct ufs_hba, eh_work);
6540
6541         dev_info(hba->dev,
6542                  "%s started; HBA state %s; powered %d; shutting down %d; saved_err = %d; saved_uic_err = %d; force_reset = %d%s\n",
6543                  __func__, ufshcd_state_name[hba->ufshcd_state],
6544                  hba->is_powered, hba->shutting_down, hba->saved_err,
6545                  hba->saved_uic_err, hba->force_reset,
6546                  ufshcd_is_link_broken(hba) ? "; link is broken" : "");
6547
6548         down(&hba->host_sem);
6549         spin_lock_irqsave(hba->host->host_lock, flags);
6550         if (ufshcd_err_handling_should_stop(hba)) {
6551                 if (hba->ufshcd_state != UFSHCD_STATE_ERROR)
6552                         hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
6553                 spin_unlock_irqrestore(hba->host->host_lock, flags);
6554                 up(&hba->host_sem);
6555                 return;
6556         }
6557         ufshcd_set_eh_in_progress(hba);
6558         spin_unlock_irqrestore(hba->host->host_lock, flags);
6559         ufshcd_err_handling_prepare(hba);
6560         /* Complete requests that have door-bell cleared by h/w */
6561         ufshcd_complete_requests(hba, false);
6562         spin_lock_irqsave(hba->host->host_lock, flags);
6563 again:
6564         needs_restore = false;
6565         needs_reset = false;
6566
6567         if (hba->ufshcd_state != UFSHCD_STATE_ERROR)
6568                 hba->ufshcd_state = UFSHCD_STATE_RESET;
6569         /*
6570          * A full reset and restore might have happened after preparation
6571          * is finished, double check whether we should stop.
6572          */
6573         if (ufshcd_err_handling_should_stop(hba))
6574                 goto skip_err_handling;
6575
6576         if ((hba->dev_quirks & UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) &&
6577             !hba->force_reset) {
6578                 bool ret;
6579
6580                 spin_unlock_irqrestore(hba->host->host_lock, flags);
6581                 /* release the lock as ufshcd_quirk_dl_nac_errors() may sleep */
6582                 ret = ufshcd_quirk_dl_nac_errors(hba);
6583                 spin_lock_irqsave(hba->host->host_lock, flags);
6584                 if (!ret && ufshcd_err_handling_should_stop(hba))
6585                         goto skip_err_handling;
6586         }
6587
6588         if ((hba->saved_err & (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK)) ||
6589             (hba->saved_uic_err &&
6590              (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) {
6591                 bool pr_prdt = !!(hba->saved_err & SYSTEM_BUS_FATAL_ERROR);
6592
6593                 spin_unlock_irqrestore(hba->host->host_lock, flags);
6594                 ufshcd_print_host_state(hba);
6595                 ufshcd_print_pwr_info(hba);
6596                 ufshcd_print_evt_hist(hba);
6597                 ufshcd_print_tmrs(hba, hba->outstanding_tasks);
6598                 ufshcd_print_trs_all(hba, pr_prdt);
6599                 spin_lock_irqsave(hba->host->host_lock, flags);
6600         }
6601
6602         /*
6603          * if host reset is required then skip clearing the pending
6604          * transfers forcefully because they will get cleared during
6605          * host reset and restore
6606          */
6607         if (hba->force_reset || ufshcd_is_link_broken(hba) ||
6608             ufshcd_is_saved_err_fatal(hba) ||
6609             ((hba->saved_err & UIC_ERROR) &&
6610              (hba->saved_uic_err & (UFSHCD_UIC_DL_NAC_RECEIVED_ERROR |
6611                                     UFSHCD_UIC_DL_TCx_REPLAY_ERROR)))) {
6612                 needs_reset = true;
6613                 goto do_reset;
6614         }
6615
6616         /*
6617          * If LINERESET was caught, UFS might have been put to PWM mode,
6618          * check if power mode restore is needed.
6619          */
6620         if (hba->saved_uic_err & UFSHCD_UIC_PA_GENERIC_ERROR) {
6621                 hba->saved_uic_err &= ~UFSHCD_UIC_PA_GENERIC_ERROR;
6622                 if (!hba->saved_uic_err)
6623                         hba->saved_err &= ~UIC_ERROR;
6624                 spin_unlock_irqrestore(hba->host->host_lock, flags);
6625                 if (ufshcd_is_pwr_mode_restore_needed(hba))
6626                         needs_restore = true;
6627                 spin_lock_irqsave(hba->host->host_lock, flags);
6628                 if (!hba->saved_err && !needs_restore)
6629                         goto skip_err_handling;
6630         }
6631
6632         hba->silence_err_logs = true;
6633         /* release lock as clear command might sleep */
6634         spin_unlock_irqrestore(hba->host->host_lock, flags);
6635
6636         needs_reset = ufshcd_abort_all(hba);
6637
6638         spin_lock_irqsave(hba->host->host_lock, flags);
6639         hba->silence_err_logs = false;
6640         if (needs_reset)
6641                 goto do_reset;
6642
6643         /*
6644          * After all reqs and tasks are cleared from doorbell,
6645          * now it is safe to retore power mode.
6646          */
6647         if (needs_restore) {
6648                 spin_unlock_irqrestore(hba->host->host_lock, flags);
6649                 /*
6650                  * Hold the scaling lock just in case dev cmds
6651                  * are sent via bsg and/or sysfs.
6652                  */
6653                 down_write(&hba->clk_scaling_lock);
6654                 hba->force_pmc = true;
6655                 pmc_err = ufshcd_config_pwr_mode(hba, &(hba->pwr_info));
6656                 if (pmc_err) {
6657                         needs_reset = true;
6658                         dev_err(hba->dev, "%s: Failed to restore power mode, err = %d\n",
6659                                         __func__, pmc_err);
6660                 }
6661                 hba->force_pmc = false;
6662                 ufshcd_print_pwr_info(hba);
6663                 up_write(&hba->clk_scaling_lock);
6664                 spin_lock_irqsave(hba->host->host_lock, flags);
6665         }
6666
6667 do_reset:
6668         /* Fatal errors need reset */
6669         if (needs_reset) {
6670                 int err;
6671
6672                 hba->force_reset = false;
6673                 spin_unlock_irqrestore(hba->host->host_lock, flags);
6674                 err = ufshcd_reset_and_restore(hba);
6675                 if (err)
6676                         dev_err(hba->dev, "%s: reset and restore failed with err %d\n",
6677                                         __func__, err);
6678                 else
6679                         ufshcd_recover_pm_error(hba);
6680                 spin_lock_irqsave(hba->host->host_lock, flags);
6681         }
6682
6683 skip_err_handling:
6684         if (!needs_reset) {
6685                 if (hba->ufshcd_state == UFSHCD_STATE_RESET)
6686                         hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
6687                 if (hba->saved_err || hba->saved_uic_err)
6688                         dev_err_ratelimited(hba->dev, "%s: exit: saved_err 0x%x saved_uic_err 0x%x",
6689                             __func__, hba->saved_err, hba->saved_uic_err);
6690         }
6691         /* Exit in an operational state or dead */
6692         if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL &&
6693             hba->ufshcd_state != UFSHCD_STATE_ERROR) {
6694                 if (--retries)
6695                         goto again;
6696                 hba->ufshcd_state = UFSHCD_STATE_ERROR;
6697         }
6698         ufshcd_clear_eh_in_progress(hba);
6699         spin_unlock_irqrestore(hba->host->host_lock, flags);
6700         ufshcd_err_handling_unprepare(hba);
6701         up(&hba->host_sem);
6702
6703         dev_info(hba->dev, "%s finished; HBA state %s\n", __func__,
6704                  ufshcd_state_name[hba->ufshcd_state]);
6705 }
6706
6707 /**
6708  * ufshcd_update_uic_error - check and set fatal UIC error flags.
6709  * @hba: per-adapter instance
6710  *
6711  * Return:
6712  *  IRQ_HANDLED - If interrupt is valid
6713  *  IRQ_NONE    - If invalid interrupt
6714  */
6715 static irqreturn_t ufshcd_update_uic_error(struct ufs_hba *hba)
6716 {
6717         u32 reg;
6718         irqreturn_t retval = IRQ_NONE;
6719
6720         /* PHY layer error */
6721         reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER);
6722         if ((reg & UIC_PHY_ADAPTER_LAYER_ERROR) &&
6723             (reg & UIC_PHY_ADAPTER_LAYER_ERROR_CODE_MASK)) {
6724                 ufshcd_update_evt_hist(hba, UFS_EVT_PA_ERR, reg);
6725                 /*
6726                  * To know whether this error is fatal or not, DB timeout
6727                  * must be checked but this error is handled separately.
6728                  */
6729                 if (reg & UIC_PHY_ADAPTER_LAYER_LANE_ERR_MASK)
6730                         dev_dbg(hba->dev, "%s: UIC Lane error reported\n",
6731                                         __func__);
6732
6733                 /* Got a LINERESET indication. */
6734                 if (reg & UIC_PHY_ADAPTER_LAYER_GENERIC_ERROR) {
6735                         struct uic_command *cmd = NULL;
6736
6737                         hba->uic_error |= UFSHCD_UIC_PA_GENERIC_ERROR;
6738                         if (hba->uic_async_done && hba->active_uic_cmd)
6739                                 cmd = hba->active_uic_cmd;
6740                         /*
6741                          * Ignore the LINERESET during power mode change
6742                          * operation via DME_SET command.
6743                          */
6744                         if (cmd && (cmd->command == UIC_CMD_DME_SET))
6745                                 hba->uic_error &= ~UFSHCD_UIC_PA_GENERIC_ERROR;
6746                 }
6747                 retval |= IRQ_HANDLED;
6748         }
6749
6750         /* PA_INIT_ERROR is fatal and needs UIC reset */
6751         reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DATA_LINK_LAYER);
6752         if ((reg & UIC_DATA_LINK_LAYER_ERROR) &&
6753             (reg & UIC_DATA_LINK_LAYER_ERROR_CODE_MASK)) {
6754                 ufshcd_update_evt_hist(hba, UFS_EVT_DL_ERR, reg);
6755
6756                 if (reg & UIC_DATA_LINK_LAYER_ERROR_PA_INIT)
6757                         hba->uic_error |= UFSHCD_UIC_DL_PA_INIT_ERROR;
6758                 else if (hba->dev_quirks &
6759                                 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
6760                         if (reg & UIC_DATA_LINK_LAYER_ERROR_NAC_RECEIVED)
6761                                 hba->uic_error |=
6762                                         UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
6763                         else if (reg & UIC_DATA_LINK_LAYER_ERROR_TCx_REPLAY_TIMEOUT)
6764                                 hba->uic_error |= UFSHCD_UIC_DL_TCx_REPLAY_ERROR;
6765                 }
6766                 retval |= IRQ_HANDLED;
6767         }
6768
6769         /* UIC NL/TL/DME errors needs software retry */
6770         reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_NETWORK_LAYER);
6771         if ((reg & UIC_NETWORK_LAYER_ERROR) &&
6772             (reg & UIC_NETWORK_LAYER_ERROR_CODE_MASK)) {
6773                 ufshcd_update_evt_hist(hba, UFS_EVT_NL_ERR, reg);
6774                 hba->uic_error |= UFSHCD_UIC_NL_ERROR;
6775                 retval |= IRQ_HANDLED;
6776         }
6777
6778         reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_TRANSPORT_LAYER);
6779         if ((reg & UIC_TRANSPORT_LAYER_ERROR) &&
6780             (reg & UIC_TRANSPORT_LAYER_ERROR_CODE_MASK)) {
6781                 ufshcd_update_evt_hist(hba, UFS_EVT_TL_ERR, reg);
6782                 hba->uic_error |= UFSHCD_UIC_TL_ERROR;
6783                 retval |= IRQ_HANDLED;
6784         }
6785
6786         reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DME);
6787         if ((reg & UIC_DME_ERROR) &&
6788             (reg & UIC_DME_ERROR_CODE_MASK)) {
6789                 ufshcd_update_evt_hist(hba, UFS_EVT_DME_ERR, reg);
6790                 hba->uic_error |= UFSHCD_UIC_DME_ERROR;
6791                 retval |= IRQ_HANDLED;
6792         }
6793
6794         dev_dbg(hba->dev, "%s: UIC error flags = 0x%08x\n",
6795                         __func__, hba->uic_error);
6796         return retval;
6797 }
6798
6799 /**
6800  * ufshcd_check_errors - Check for errors that need s/w attention
6801  * @hba: per-adapter instance
6802  * @intr_status: interrupt status generated by the controller
6803  *
6804  * Return:
6805  *  IRQ_HANDLED - If interrupt is valid
6806  *  IRQ_NONE    - If invalid interrupt
6807  */
6808 static irqreturn_t ufshcd_check_errors(struct ufs_hba *hba, u32 intr_status)
6809 {
6810         bool queue_eh_work = false;
6811         irqreturn_t retval = IRQ_NONE;
6812
6813         spin_lock(hba->host->host_lock);
6814         hba->errors |= UFSHCD_ERROR_MASK & intr_status;
6815
6816         if (hba->errors & INT_FATAL_ERRORS) {
6817                 ufshcd_update_evt_hist(hba, UFS_EVT_FATAL_ERR,
6818                                        hba->errors);
6819                 queue_eh_work = true;
6820         }
6821
6822         if (hba->errors & UIC_ERROR) {
6823                 hba->uic_error = 0;
6824                 retval = ufshcd_update_uic_error(hba);
6825                 if (hba->uic_error)
6826                         queue_eh_work = true;
6827         }
6828
6829         if (hba->errors & UFSHCD_UIC_HIBERN8_MASK) {
6830                 dev_err(hba->dev,
6831                         "%s: Auto Hibern8 %s failed - status: 0x%08x, upmcrs: 0x%08x\n",
6832                         __func__, (hba->errors & UIC_HIBERNATE_ENTER) ?
6833                         "Enter" : "Exit",
6834                         hba->errors, ufshcd_get_upmcrs(hba));
6835                 ufshcd_update_evt_hist(hba, UFS_EVT_AUTO_HIBERN8_ERR,
6836                                        hba->errors);
6837                 ufshcd_set_link_broken(hba);
6838                 queue_eh_work = true;
6839         }
6840
6841         if (queue_eh_work) {
6842                 /*
6843                  * update the transfer error masks to sticky bits, let's do this
6844                  * irrespective of current ufshcd_state.
6845                  */
6846                 hba->saved_err |= hba->errors;
6847                 hba->saved_uic_err |= hba->uic_error;
6848
6849                 /* dump controller state before resetting */
6850                 if ((hba->saved_err &
6851                      (INT_FATAL_ERRORS | UFSHCD_UIC_HIBERN8_MASK)) ||
6852                     (hba->saved_uic_err &&
6853                      (hba->saved_uic_err != UFSHCD_UIC_PA_GENERIC_ERROR))) {
6854                         dev_err(hba->dev, "%s: saved_err 0x%x saved_uic_err 0x%x\n",
6855                                         __func__, hba->saved_err,
6856                                         hba->saved_uic_err);
6857                         ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE,
6858                                          "host_regs: ");
6859                         ufshcd_print_pwr_info(hba);
6860                 }
6861                 ufshcd_schedule_eh_work(hba);
6862                 retval |= IRQ_HANDLED;
6863         }
6864         /*
6865          * if (!queue_eh_work) -
6866          * Other errors are either non-fatal where host recovers
6867          * itself without s/w intervention or errors that will be
6868          * handled by the SCSI core layer.
6869          */
6870         hba->errors = 0;
6871         hba->uic_error = 0;
6872         spin_unlock(hba->host->host_lock);
6873         return retval;
6874 }
6875
6876 /**
6877  * ufshcd_tmc_handler - handle task management function completion
6878  * @hba: per adapter instance
6879  *
6880  * Return:
6881  *  IRQ_HANDLED - If interrupt is valid
6882  *  IRQ_NONE    - If invalid interrupt
6883  */
6884 static irqreturn_t ufshcd_tmc_handler(struct ufs_hba *hba)
6885 {
6886         unsigned long flags, pending, issued;
6887         irqreturn_t ret = IRQ_NONE;
6888         int tag;
6889
6890         spin_lock_irqsave(hba->host->host_lock, flags);
6891         pending = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
6892         issued = hba->outstanding_tasks & ~pending;
6893         for_each_set_bit(tag, &issued, hba->nutmrs) {
6894                 struct request *req = hba->tmf_rqs[tag];
6895                 struct completion *c = req->end_io_data;
6896
6897                 complete(c);
6898                 ret = IRQ_HANDLED;
6899         }
6900         spin_unlock_irqrestore(hba->host->host_lock, flags);
6901
6902         return ret;
6903 }
6904
6905 /**
6906  * ufshcd_handle_mcq_cq_events - handle MCQ completion queue events
6907  * @hba: per adapter instance
6908  *
6909  * Return: IRQ_HANDLED if interrupt is handled.
6910  */
6911 static irqreturn_t ufshcd_handle_mcq_cq_events(struct ufs_hba *hba)
6912 {
6913         struct ufs_hw_queue *hwq;
6914         unsigned long outstanding_cqs;
6915         unsigned int nr_queues;
6916         int i, ret;
6917         u32 events;
6918
6919         ret = ufshcd_vops_get_outstanding_cqs(hba, &outstanding_cqs);
6920         if (ret)
6921                 outstanding_cqs = (1U << hba->nr_hw_queues) - 1;
6922
6923         /* Exclude the poll queues */
6924         nr_queues = hba->nr_hw_queues - hba->nr_queues[HCTX_TYPE_POLL];
6925         for_each_set_bit(i, &outstanding_cqs, nr_queues) {
6926                 hwq = &hba->uhq[i];
6927
6928                 events = ufshcd_mcq_read_cqis(hba, i);
6929                 if (events)
6930                         ufshcd_mcq_write_cqis(hba, events, i);
6931
6932                 if (events & UFSHCD_MCQ_CQIS_TAIL_ENT_PUSH_STS)
6933                         ufshcd_mcq_poll_cqe_lock(hba, hwq);
6934         }
6935
6936         return IRQ_HANDLED;
6937 }
6938
6939 /**
6940  * ufshcd_sl_intr - Interrupt service routine
6941  * @hba: per adapter instance
6942  * @intr_status: contains interrupts generated by the controller
6943  *
6944  * Return:
6945  *  IRQ_HANDLED - If interrupt is valid
6946  *  IRQ_NONE    - If invalid interrupt
6947  */
6948 static irqreturn_t ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status)
6949 {
6950         irqreturn_t retval = IRQ_NONE;
6951
6952         if (intr_status & UFSHCD_UIC_MASK)
6953                 retval |= ufshcd_uic_cmd_compl(hba, intr_status);
6954
6955         if (intr_status & UFSHCD_ERROR_MASK || hba->errors)
6956                 retval |= ufshcd_check_errors(hba, intr_status);
6957
6958         if (intr_status & UTP_TASK_REQ_COMPL)
6959                 retval |= ufshcd_tmc_handler(hba);
6960
6961         if (intr_status & UTP_TRANSFER_REQ_COMPL)
6962                 retval |= ufshcd_transfer_req_compl(hba);
6963
6964         if (intr_status & MCQ_CQ_EVENT_STATUS)
6965                 retval |= ufshcd_handle_mcq_cq_events(hba);
6966
6967         return retval;
6968 }
6969
6970 /**
6971  * ufshcd_intr - Main interrupt service routine
6972  * @irq: irq number
6973  * @__hba: pointer to adapter instance
6974  *
6975  * Return:
6976  *  IRQ_HANDLED - If interrupt is valid
6977  *  IRQ_NONE    - If invalid interrupt
6978  */
6979 static irqreturn_t ufshcd_intr(int irq, void *__hba)
6980 {
6981         u32 intr_status, enabled_intr_status = 0;
6982         irqreturn_t retval = IRQ_NONE;
6983         struct ufs_hba *hba = __hba;
6984         int retries = hba->nutrs;
6985
6986         intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
6987         hba->ufs_stats.last_intr_status = intr_status;
6988         hba->ufs_stats.last_intr_ts = local_clock();
6989
6990         /*
6991          * There could be max of hba->nutrs reqs in flight and in worst case
6992          * if the reqs get finished 1 by 1 after the interrupt status is
6993          * read, make sure we handle them by checking the interrupt status
6994          * again in a loop until we process all of the reqs before returning.
6995          */
6996         while (intr_status && retries--) {
6997                 enabled_intr_status =
6998                         intr_status & ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
6999                 ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS);
7000                 if (enabled_intr_status)
7001                         retval |= ufshcd_sl_intr(hba, enabled_intr_status);
7002
7003                 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
7004         }
7005
7006         if (enabled_intr_status && retval == IRQ_NONE &&
7007             (!(enabled_intr_status & UTP_TRANSFER_REQ_COMPL) ||
7008              hba->outstanding_reqs) && !ufshcd_eh_in_progress(hba)) {
7009                 dev_err(hba->dev, "%s: Unhandled interrupt 0x%08x (0x%08x, 0x%08x)\n",
7010                                         __func__,
7011                                         intr_status,
7012                                         hba->ufs_stats.last_intr_status,
7013                                         enabled_intr_status);
7014                 ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
7015         }
7016
7017         return retval;
7018 }
7019
7020 static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag)
7021 {
7022         int err = 0;
7023         u32 mask = 1 << tag;
7024
7025         if (!test_bit(tag, &hba->outstanding_tasks))
7026                 goto out;
7027
7028         ufshcd_utmrl_clear(hba, tag);
7029
7030         /* poll for max. 1 sec to clear door bell register by h/w */
7031         err = ufshcd_wait_for_register(hba,
7032                         REG_UTP_TASK_REQ_DOOR_BELL,
7033                         mask, 0, 1000, 1000);
7034
7035         dev_err(hba->dev, "Clearing task management function with tag %d %s\n",
7036                 tag, err < 0 ? "failed" : "succeeded");
7037
7038 out:
7039         return err;
7040 }
7041
7042 static int __ufshcd_issue_tm_cmd(struct ufs_hba *hba,
7043                 struct utp_task_req_desc *treq, u8 tm_function)
7044 {
7045         struct request_queue *q = hba->tmf_queue;
7046         struct Scsi_Host *host = hba->host;
7047         DECLARE_COMPLETION_ONSTACK(wait);
7048         struct request *req;
7049         unsigned long flags;
7050         int task_tag, err;
7051
7052         /*
7053          * blk_mq_alloc_request() is used here only to get a free tag.
7054          */
7055         req = blk_mq_alloc_request(q, REQ_OP_DRV_OUT, 0);
7056         if (IS_ERR(req))
7057                 return PTR_ERR(req);
7058
7059         req->end_io_data = &wait;
7060         ufshcd_hold(hba);
7061
7062         spin_lock_irqsave(host->host_lock, flags);
7063
7064         task_tag = req->tag;
7065         hba->tmf_rqs[req->tag] = req;
7066         treq->upiu_req.req_header.task_tag = task_tag;
7067
7068         memcpy(hba->utmrdl_base_addr + task_tag, treq, sizeof(*treq));
7069         ufshcd_vops_setup_task_mgmt(hba, task_tag, tm_function);
7070
7071         __set_bit(task_tag, &hba->outstanding_tasks);
7072
7073         spin_unlock_irqrestore(host->host_lock, flags);
7074
7075         /* send command to the controller */
7076         ufshcd_writel(hba, 1 << task_tag, REG_UTP_TASK_REQ_DOOR_BELL);
7077
7078         ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_SEND);
7079
7080         /* wait until the task management command is completed */
7081         err = wait_for_completion_io_timeout(&wait,
7082                         msecs_to_jiffies(TM_CMD_TIMEOUT));
7083         if (!err) {
7084                 ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_ERR);
7085                 dev_err(hba->dev, "%s: task management cmd 0x%.2x timed-out\n",
7086                                 __func__, tm_function);
7087                 if (ufshcd_clear_tm_cmd(hba, task_tag))
7088                         dev_WARN(hba->dev, "%s: unable to clear tm cmd (slot %d) after timeout\n",
7089                                         __func__, task_tag);
7090                 err = -ETIMEDOUT;
7091         } else {
7092                 err = 0;
7093                 memcpy(treq, hba->utmrdl_base_addr + task_tag, sizeof(*treq));
7094
7095                 ufshcd_add_tm_upiu_trace(hba, task_tag, UFS_TM_COMP);
7096         }
7097
7098         spin_lock_irqsave(hba->host->host_lock, flags);
7099         hba->tmf_rqs[req->tag] = NULL;
7100         __clear_bit(task_tag, &hba->outstanding_tasks);
7101         spin_unlock_irqrestore(hba->host->host_lock, flags);
7102
7103         ufshcd_release(hba);
7104         blk_mq_free_request(req);
7105
7106         return err;
7107 }
7108
7109 /**
7110  * ufshcd_issue_tm_cmd - issues task management commands to controller
7111  * @hba: per adapter instance
7112  * @lun_id: LUN ID to which TM command is sent
7113  * @task_id: task ID to which the TM command is applicable
7114  * @tm_function: task management function opcode
7115  * @tm_response: task management service response return value
7116  *
7117  * Return: non-zero value on error, zero on success.
7118  */
7119 static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id,
7120                 u8 tm_function, u8 *tm_response)
7121 {
7122         struct utp_task_req_desc treq = { };
7123         enum utp_ocs ocs_value;
7124         int err;
7125
7126         /* Configure task request descriptor */
7127         treq.header.interrupt = 1;
7128         treq.header.ocs = OCS_INVALID_COMMAND_STATUS;
7129
7130         /* Configure task request UPIU */
7131         treq.upiu_req.req_header.transaction_code = UPIU_TRANSACTION_TASK_REQ;
7132         treq.upiu_req.req_header.lun = lun_id;
7133         treq.upiu_req.req_header.tm_function = tm_function;
7134
7135         /*
7136          * The host shall provide the same value for LUN field in the basic
7137          * header and for Input Parameter.
7138          */
7139         treq.upiu_req.input_param1 = cpu_to_be32(lun_id);
7140         treq.upiu_req.input_param2 = cpu_to_be32(task_id);
7141
7142         err = __ufshcd_issue_tm_cmd(hba, &treq, tm_function);
7143         if (err == -ETIMEDOUT)
7144                 return err;
7145
7146         ocs_value = treq.header.ocs & MASK_OCS;
7147         if (ocs_value != OCS_SUCCESS)
7148                 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n",
7149                                 __func__, ocs_value);
7150         else if (tm_response)
7151                 *tm_response = be32_to_cpu(treq.upiu_rsp.output_param1) &
7152                                 MASK_TM_SERVICE_RESP;
7153         return err;
7154 }
7155
7156 /**
7157  * ufshcd_issue_devman_upiu_cmd - API for sending "utrd" type requests
7158  * @hba:        per-adapter instance
7159  * @req_upiu:   upiu request
7160  * @rsp_upiu:   upiu reply
7161  * @desc_buff:  pointer to descriptor buffer, NULL if NA
7162  * @buff_len:   descriptor size, 0 if NA
7163  * @cmd_type:   specifies the type (NOP, Query...)
7164  * @desc_op:    descriptor operation
7165  *
7166  * Those type of requests uses UTP Transfer Request Descriptor - utrd.
7167  * Therefore, it "rides" the device management infrastructure: uses its tag and
7168  * tasks work queues.
7169  *
7170  * Since there is only one available tag for device management commands,
7171  * the caller is expected to hold the hba->dev_cmd.lock mutex.
7172  *
7173  * Return: 0 upon success; < 0 upon failure.
7174  */
7175 static int ufshcd_issue_devman_upiu_cmd(struct ufs_hba *hba,
7176                                         struct utp_upiu_req *req_upiu,
7177                                         struct utp_upiu_req *rsp_upiu,
7178                                         u8 *desc_buff, int *buff_len,
7179                                         enum dev_cmd_type cmd_type,
7180                                         enum query_opcode desc_op)
7181 {
7182         const u32 tag = hba->reserved_slot;
7183         struct ufshcd_lrb *lrbp = &hba->lrb[tag];
7184         int err = 0;
7185         u8 upiu_flags;
7186
7187         /* Protects use of hba->reserved_slot. */
7188         lockdep_assert_held(&hba->dev_cmd.lock);
7189
7190         ufshcd_setup_dev_cmd(hba, lrbp, cmd_type, 0, tag);
7191
7192         ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, 0);
7193
7194         /* update the task tag in the request upiu */
7195         req_upiu->header.task_tag = tag;
7196
7197         /* just copy the upiu request as it is */
7198         memcpy(lrbp->ucd_req_ptr, req_upiu, sizeof(*lrbp->ucd_req_ptr));
7199         if (desc_buff && desc_op == UPIU_QUERY_OPCODE_WRITE_DESC) {
7200                 /* The Data Segment Area is optional depending upon the query
7201                  * function value. for WRITE DESCRIPTOR, the data segment
7202                  * follows right after the tsf.
7203                  */
7204                 memcpy(lrbp->ucd_req_ptr + 1, desc_buff, *buff_len);
7205                 *buff_len = 0;
7206         }
7207
7208         memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
7209
7210         /*
7211          * ignore the returning value here - ufshcd_check_query_response is
7212          * bound to fail since dev_cmd.query and dev_cmd.type were left empty.
7213          * read the response directly ignoring all errors.
7214          */
7215         ufshcd_issue_dev_cmd(hba, lrbp, tag, QUERY_REQ_TIMEOUT);
7216
7217         /* just copy the upiu response as it is */
7218         memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu));
7219         if (desc_buff && desc_op == UPIU_QUERY_OPCODE_READ_DESC) {
7220                 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr + sizeof(*rsp_upiu);
7221                 u16 resp_len = be16_to_cpu(lrbp->ucd_rsp_ptr->header
7222                                            .data_segment_length);
7223
7224                 if (*buff_len >= resp_len) {
7225                         memcpy(desc_buff, descp, resp_len);
7226                         *buff_len = resp_len;
7227                 } else {
7228                         dev_warn(hba->dev,
7229                                  "%s: rsp size %d is bigger than buffer size %d",
7230                                  __func__, resp_len, *buff_len);
7231                         *buff_len = 0;
7232                         err = -EINVAL;
7233                 }
7234         }
7235         ufshcd_add_query_upiu_trace(hba, err ? UFS_QUERY_ERR : UFS_QUERY_COMP,
7236                                     (struct utp_upiu_req *)lrbp->ucd_rsp_ptr);
7237
7238         return err;
7239 }
7240
7241 /**
7242  * ufshcd_exec_raw_upiu_cmd - API function for sending raw upiu commands
7243  * @hba:        per-adapter instance
7244  * @req_upiu:   upiu request
7245  * @rsp_upiu:   upiu reply - only 8 DW as we do not support scsi commands
7246  * @msgcode:    message code, one of UPIU Transaction Codes Initiator to Target
7247  * @desc_buff:  pointer to descriptor buffer, NULL if NA
7248  * @buff_len:   descriptor size, 0 if NA
7249  * @desc_op:    descriptor operation
7250  *
7251  * Supports UTP Transfer requests (nop and query), and UTP Task
7252  * Management requests.
7253  * It is up to the caller to fill the upiu conent properly, as it will
7254  * be copied without any further input validations.
7255  *
7256  * Return: 0 upon success; < 0 upon failure.
7257  */
7258 int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba,
7259                              struct utp_upiu_req *req_upiu,
7260                              struct utp_upiu_req *rsp_upiu,
7261                              enum upiu_request_transaction msgcode,
7262                              u8 *desc_buff, int *buff_len,
7263                              enum query_opcode desc_op)
7264 {
7265         int err;
7266         enum dev_cmd_type cmd_type = DEV_CMD_TYPE_QUERY;
7267         struct utp_task_req_desc treq = { };
7268         enum utp_ocs ocs_value;
7269         u8 tm_f = req_upiu->header.tm_function;
7270
7271         switch (msgcode) {
7272         case UPIU_TRANSACTION_NOP_OUT:
7273                 cmd_type = DEV_CMD_TYPE_NOP;
7274                 fallthrough;
7275         case UPIU_TRANSACTION_QUERY_REQ:
7276                 ufshcd_dev_man_lock(hba);
7277                 err = ufshcd_issue_devman_upiu_cmd(hba, req_upiu, rsp_upiu,
7278                                                    desc_buff, buff_len,
7279                                                    cmd_type, desc_op);
7280                 ufshcd_dev_man_unlock(hba);
7281
7282                 break;
7283         case UPIU_TRANSACTION_TASK_REQ:
7284                 treq.header.interrupt = 1;
7285                 treq.header.ocs = OCS_INVALID_COMMAND_STATUS;
7286
7287                 memcpy(&treq.upiu_req, req_upiu, sizeof(*req_upiu));
7288
7289                 err = __ufshcd_issue_tm_cmd(hba, &treq, tm_f);
7290                 if (err == -ETIMEDOUT)
7291                         break;
7292
7293                 ocs_value = treq.header.ocs & MASK_OCS;
7294                 if (ocs_value != OCS_SUCCESS) {
7295                         dev_err(hba->dev, "%s: failed, ocs = 0x%x\n", __func__,
7296                                 ocs_value);
7297                         break;
7298                 }
7299
7300                 memcpy(rsp_upiu, &treq.upiu_rsp, sizeof(*rsp_upiu));
7301
7302                 break;
7303         default:
7304                 err = -EINVAL;
7305
7306                 break;
7307         }
7308
7309         return err;
7310 }
7311
7312 /**
7313  * ufshcd_advanced_rpmb_req_handler - handle advanced RPMB request
7314  * @hba:        per adapter instance
7315  * @req_upiu:   upiu request
7316  * @rsp_upiu:   upiu reply
7317  * @req_ehs:    EHS field which contains Advanced RPMB Request Message
7318  * @rsp_ehs:    EHS field which returns Advanced RPMB Response Message
7319  * @sg_cnt:     The number of sg lists actually used
7320  * @sg_list:    Pointer to SG list when DATA IN/OUT UPIU is required in ARPMB operation
7321  * @dir:        DMA direction
7322  *
7323  * Return: zero on success, non-zero on failure.
7324  */
7325 int ufshcd_advanced_rpmb_req_handler(struct ufs_hba *hba, struct utp_upiu_req *req_upiu,
7326                          struct utp_upiu_req *rsp_upiu, struct ufs_ehs *req_ehs,
7327                          struct ufs_ehs *rsp_ehs, int sg_cnt, struct scatterlist *sg_list,
7328                          enum dma_data_direction dir)
7329 {
7330         const u32 tag = hba->reserved_slot;
7331         struct ufshcd_lrb *lrbp = &hba->lrb[tag];
7332         int err = 0;
7333         int result;
7334         u8 upiu_flags;
7335         u8 *ehs_data;
7336         u16 ehs_len;
7337         int ehs = (hba->capabilities & MASK_EHSLUTRD_SUPPORTED) ? 2 : 0;
7338
7339         /* Protects use of hba->reserved_slot. */
7340         ufshcd_dev_man_lock(hba);
7341
7342         ufshcd_setup_dev_cmd(hba, lrbp, DEV_CMD_TYPE_RPMB, UFS_UPIU_RPMB_WLUN, tag);
7343
7344         ufshcd_prepare_req_desc_hdr(hba, lrbp, &upiu_flags, DMA_NONE, ehs);
7345
7346         /* update the task tag */
7347         req_upiu->header.task_tag = tag;
7348
7349         /* copy the UPIU(contains CDB) request as it is */
7350         memcpy(lrbp->ucd_req_ptr, req_upiu, sizeof(*lrbp->ucd_req_ptr));
7351         /* Copy EHS, starting with byte32, immediately after the CDB package */
7352         memcpy(lrbp->ucd_req_ptr + 1, req_ehs, sizeof(*req_ehs));
7353
7354         if (dir != DMA_NONE && sg_list)
7355                 ufshcd_sgl_to_prdt(hba, lrbp, sg_cnt, sg_list);
7356
7357         memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
7358
7359         err = ufshcd_issue_dev_cmd(hba, lrbp, tag, ADVANCED_RPMB_REQ_TIMEOUT);
7360
7361         if (!err) {
7362                 /* Just copy the upiu response as it is */
7363                 memcpy(rsp_upiu, lrbp->ucd_rsp_ptr, sizeof(*rsp_upiu));
7364                 /* Get the response UPIU result */
7365                 result = (lrbp->ucd_rsp_ptr->header.response << 8) |
7366                         lrbp->ucd_rsp_ptr->header.status;
7367
7368                 ehs_len = lrbp->ucd_rsp_ptr->header.ehs_length;
7369                 /*
7370                  * Since the bLength in EHS indicates the total size of the EHS Header and EHS Data
7371                  * in 32 Byte units, the value of the bLength Request/Response for Advanced RPMB
7372                  * Message is 02h
7373                  */
7374                 if (ehs_len == 2 && rsp_ehs) {
7375                         /*
7376                          * ucd_rsp_ptr points to a buffer with a length of 512 bytes
7377                          * (ALIGNED_UPIU_SIZE = 512), and the EHS data just starts from byte32
7378                          */
7379                         ehs_data = (u8 *)lrbp->ucd_rsp_ptr + EHS_OFFSET_IN_RESPONSE;
7380                         memcpy(rsp_ehs, ehs_data, ehs_len * 32);
7381                 }
7382         }
7383
7384         ufshcd_dev_man_unlock(hba);
7385
7386         return err ? : result;
7387 }
7388
7389 /**
7390  * ufshcd_eh_device_reset_handler() - Reset a single logical unit.
7391  * @cmd: SCSI command pointer
7392  *
7393  * Return: SUCCESS or FAILED.
7394  */
7395 static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd)
7396 {
7397         unsigned long flags, pending_reqs = 0, not_cleared = 0;
7398         struct Scsi_Host *host;
7399         struct ufs_hba *hba;
7400         struct ufs_hw_queue *hwq;
7401         struct ufshcd_lrb *lrbp;
7402         u32 pos, not_cleared_mask = 0;
7403         int err;
7404         u8 resp = 0xF, lun;
7405
7406         host = cmd->device->host;
7407         hba = shost_priv(host);
7408
7409         lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
7410         err = ufshcd_issue_tm_cmd(hba, lun, 0, UFS_LOGICAL_RESET, &resp);
7411         if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
7412                 if (!err)
7413                         err = resp;
7414                 goto out;
7415         }
7416
7417         if (hba->mcq_enabled) {
7418                 for (pos = 0; pos < hba->nutrs; pos++) {
7419                         lrbp = &hba->lrb[pos];
7420                         if (ufshcd_cmd_inflight(lrbp->cmd) &&
7421                             lrbp->lun == lun) {
7422                                 ufshcd_clear_cmd(hba, pos);
7423                                 hwq = ufshcd_mcq_req_to_hwq(hba, scsi_cmd_to_rq(lrbp->cmd));
7424                                 ufshcd_mcq_poll_cqe_lock(hba, hwq);
7425                         }
7426                 }
7427                 err = 0;
7428                 goto out;
7429         }
7430
7431         /* clear the commands that were pending for corresponding LUN */
7432         spin_lock_irqsave(&hba->outstanding_lock, flags);
7433         for_each_set_bit(pos, &hba->outstanding_reqs, hba->nutrs)
7434                 if (hba->lrb[pos].lun == lun)
7435                         __set_bit(pos, &pending_reqs);
7436         hba->outstanding_reqs &= ~pending_reqs;
7437         spin_unlock_irqrestore(&hba->outstanding_lock, flags);
7438
7439         for_each_set_bit(pos, &pending_reqs, hba->nutrs) {
7440                 if (ufshcd_clear_cmd(hba, pos) < 0) {
7441                         spin_lock_irqsave(&hba->outstanding_lock, flags);
7442                         not_cleared = 1U << pos &
7443                                 ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
7444                         hba->outstanding_reqs |= not_cleared;
7445                         not_cleared_mask |= not_cleared;
7446                         spin_unlock_irqrestore(&hba->outstanding_lock, flags);
7447
7448                         dev_err(hba->dev, "%s: failed to clear request %d\n",
7449                                 __func__, pos);
7450                 }
7451         }
7452         __ufshcd_transfer_req_compl(hba, pending_reqs & ~not_cleared_mask);
7453
7454 out:
7455         hba->req_abort_count = 0;
7456         ufshcd_update_evt_hist(hba, UFS_EVT_DEV_RESET, (u32)err);
7457         if (!err) {
7458                 err = SUCCESS;
7459         } else {
7460                 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
7461                 err = FAILED;
7462         }
7463         return err;
7464 }
7465
7466 static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap)
7467 {
7468         struct ufshcd_lrb *lrbp;
7469         int tag;
7470
7471         for_each_set_bit(tag, &bitmap, hba->nutrs) {
7472                 lrbp = &hba->lrb[tag];
7473                 lrbp->req_abort_skip = true;
7474         }
7475 }
7476
7477 /**
7478  * ufshcd_try_to_abort_task - abort a specific task
7479  * @hba: Pointer to adapter instance
7480  * @tag: Task tag/index to be aborted
7481  *
7482  * Abort the pending command in device by sending UFS_ABORT_TASK task management
7483  * command, and in host controller by clearing the door-bell register. There can
7484  * be race between controller sending the command to the device while abort is
7485  * issued. To avoid that, first issue UFS_QUERY_TASK to check if the command is
7486  * really issued and then try to abort it.
7487  *
7488  * Return: zero on success, non-zero on failure.
7489  */
7490 int ufshcd_try_to_abort_task(struct ufs_hba *hba, int tag)
7491 {
7492         struct ufshcd_lrb *lrbp = &hba->lrb[tag];
7493         int err;
7494         int poll_cnt;
7495         u8 resp = 0xF;
7496
7497         for (poll_cnt = 100; poll_cnt; poll_cnt--) {
7498                 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
7499                                 UFS_QUERY_TASK, &resp);
7500                 if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED) {
7501                         /* cmd pending in the device */
7502                         dev_err(hba->dev, "%s: cmd pending in the device. tag = %d\n",
7503                                 __func__, tag);
7504                         break;
7505                 } else if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
7506                         /*
7507                          * cmd not pending in the device, check if it is
7508                          * in transition.
7509                          */
7510                         dev_info(
7511                                 hba->dev,
7512                                 "%s: cmd with tag %d not pending in the device.\n",
7513                                 __func__, tag);
7514                         if (!ufshcd_cmd_inflight(lrbp->cmd)) {
7515                                 dev_info(hba->dev,
7516                                          "%s: cmd with tag=%d completed.\n",
7517                                          __func__, tag);
7518                                 return 0;
7519                         }
7520                         usleep_range(100, 200);
7521                 } else {
7522                         dev_err(hba->dev,
7523                                 "%s: no response from device. tag = %d, err %d\n",
7524                                 __func__, tag, err);
7525                         return err ? : resp;
7526                 }
7527         }
7528
7529         if (!poll_cnt)
7530                 return -EBUSY;
7531
7532         err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
7533                         UFS_ABORT_TASK, &resp);
7534         if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
7535                 if (!err) {
7536                         err = resp; /* service response error */
7537                         dev_err(hba->dev, "%s: issued. tag = %d, err %d\n",
7538                                 __func__, tag, err);
7539                 }
7540                 return err;
7541         }
7542
7543         err = ufshcd_clear_cmd(hba, tag);
7544         if (err)
7545                 dev_err(hba->dev, "%s: Failed clearing cmd at tag %d, err %d\n",
7546                         __func__, tag, err);
7547
7548         return err;
7549 }
7550
7551 /**
7552  * ufshcd_abort - scsi host template eh_abort_handler callback
7553  * @cmd: SCSI command pointer
7554  *
7555  * Return: SUCCESS or FAILED.
7556  */
7557 static int ufshcd_abort(struct scsi_cmnd *cmd)
7558 {
7559         struct Scsi_Host *host = cmd->device->host;
7560         struct ufs_hba *hba = shost_priv(host);
7561         int tag = scsi_cmd_to_rq(cmd)->tag;
7562         struct ufshcd_lrb *lrbp = &hba->lrb[tag];
7563         unsigned long flags;
7564         int err = FAILED;
7565         bool outstanding;
7566         u32 reg;
7567
7568         ufshcd_hold(hba);
7569
7570         if (!hba->mcq_enabled) {
7571                 reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
7572                 if (!test_bit(tag, &hba->outstanding_reqs)) {
7573                         /* If command is already aborted/completed, return FAILED. */
7574                         dev_err(hba->dev,
7575                                 "%s: cmd at tag %d already completed, outstanding=0x%lx, doorbell=0x%x\n",
7576                                 __func__, tag, hba->outstanding_reqs, reg);
7577                         goto release;
7578                 }
7579         }
7580
7581         /* Print Transfer Request of aborted task */
7582         dev_info(hba->dev, "%s: Device abort task at tag %d\n", __func__, tag);
7583
7584         /*
7585          * Print detailed info about aborted request.
7586          * As more than one request might get aborted at the same time,
7587          * print full information only for the first aborted request in order
7588          * to reduce repeated printouts. For other aborted requests only print
7589          * basic details.
7590          */
7591         scsi_print_command(cmd);
7592         if (!hba->req_abort_count) {
7593                 ufshcd_update_evt_hist(hba, UFS_EVT_ABORT, tag);
7594                 ufshcd_print_evt_hist(hba);
7595                 ufshcd_print_host_state(hba);
7596                 ufshcd_print_pwr_info(hba);
7597                 ufshcd_print_tr(hba, tag, true);
7598         } else {
7599                 ufshcd_print_tr(hba, tag, false);
7600         }
7601         hba->req_abort_count++;
7602
7603         if (!hba->mcq_enabled && !(reg & (1 << tag))) {
7604                 /* only execute this code in single doorbell mode */
7605                 dev_err(hba->dev,
7606                 "%s: cmd was completed, but without a notifying intr, tag = %d",
7607                 __func__, tag);
7608                 __ufshcd_transfer_req_compl(hba, 1UL << tag);
7609                 goto release;
7610         }
7611
7612         /*
7613          * Task abort to the device W-LUN is illegal. When this command
7614          * will fail, due to spec violation, scsi err handling next step
7615          * will be to send LU reset which, again, is a spec violation.
7616          * To avoid these unnecessary/illegal steps, first we clean up
7617          * the lrb taken by this cmd and re-set it in outstanding_reqs,
7618          * then queue the eh_work and bail.
7619          */
7620         if (lrbp->lun == UFS_UPIU_UFS_DEVICE_WLUN) {
7621                 ufshcd_update_evt_hist(hba, UFS_EVT_ABORT, lrbp->lun);
7622
7623                 spin_lock_irqsave(host->host_lock, flags);
7624                 hba->force_reset = true;
7625                 ufshcd_schedule_eh_work(hba);
7626                 spin_unlock_irqrestore(host->host_lock, flags);
7627                 goto release;
7628         }
7629
7630         if (hba->mcq_enabled) {
7631                 /* MCQ mode. Branch off to handle abort for mcq mode */
7632                 err = ufshcd_mcq_abort(cmd);
7633                 goto release;
7634         }
7635
7636         /* Skip task abort in case previous aborts failed and report failure */
7637         if (lrbp->req_abort_skip) {
7638                 dev_err(hba->dev, "%s: skipping abort\n", __func__);
7639                 ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs);
7640                 goto release;
7641         }
7642
7643         err = ufshcd_try_to_abort_task(hba, tag);
7644         if (err) {
7645                 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
7646                 ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs);
7647                 err = FAILED;
7648                 goto release;
7649         }
7650
7651         /*
7652          * Clear the corresponding bit from outstanding_reqs since the command
7653          * has been aborted successfully.
7654          */
7655         spin_lock_irqsave(&hba->outstanding_lock, flags);
7656         outstanding = __test_and_clear_bit(tag, &hba->outstanding_reqs);
7657         spin_unlock_irqrestore(&hba->outstanding_lock, flags);
7658
7659         if (outstanding)
7660                 ufshcd_release_scsi_cmd(hba, lrbp);
7661
7662         err = SUCCESS;
7663
7664 release:
7665         /* Matches the ufshcd_hold() call at the start of this function. */
7666         ufshcd_release(hba);
7667         return err;
7668 }
7669
7670 /**
7671  * ufshcd_process_probe_result - Process the ufshcd_probe_hba() result.
7672  * @hba: UFS host controller instance.
7673  * @probe_start: time when the ufshcd_probe_hba() call started.
7674  * @ret: ufshcd_probe_hba() return value.
7675  */
7676 static void ufshcd_process_probe_result(struct ufs_hba *hba,
7677                                         ktime_t probe_start, int ret)
7678 {
7679         unsigned long flags;
7680
7681         spin_lock_irqsave(hba->host->host_lock, flags);
7682         if (ret)
7683                 hba->ufshcd_state = UFSHCD_STATE_ERROR;
7684         else if (hba->ufshcd_state == UFSHCD_STATE_RESET)
7685                 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
7686         spin_unlock_irqrestore(hba->host->host_lock, flags);
7687
7688         trace_ufshcd_init(dev_name(hba->dev), ret,
7689                           ktime_to_us(ktime_sub(ktime_get(), probe_start)),
7690                           hba->curr_dev_pwr_mode, hba->uic_link_state);
7691 }
7692
7693 /**
7694  * ufshcd_host_reset_and_restore - reset and restore host controller
7695  * @hba: per-adapter instance
7696  *
7697  * Note that host controller reset may issue DME_RESET to
7698  * local and remote (device) Uni-Pro stack and the attributes
7699  * are reset to default state.
7700  *
7701  * Return: zero on success, non-zero on failure.
7702  */
7703 static int ufshcd_host_reset_and_restore(struct ufs_hba *hba)
7704 {
7705         int err;
7706
7707         /*
7708          * Stop the host controller and complete the requests
7709          * cleared by h/w
7710          */
7711         ufshcd_hba_stop(hba);
7712         hba->silence_err_logs = true;
7713         ufshcd_complete_requests(hba, true);
7714         hba->silence_err_logs = false;
7715
7716         /* scale up clocks to max frequency before full reinitialization */
7717         ufshcd_scale_clks(hba, ULONG_MAX, true);
7718
7719         err = ufshcd_hba_enable(hba);
7720
7721         /* Establish the link again and restore the device */
7722         if (!err) {
7723                 ktime_t probe_start = ktime_get();
7724
7725                 err = ufshcd_device_init(hba, /*init_dev_params=*/false);
7726                 if (!err)
7727                         err = ufshcd_probe_hba(hba, false);
7728                 ufshcd_process_probe_result(hba, probe_start, err);
7729         }
7730
7731         if (err)
7732                 dev_err(hba->dev, "%s: Host init failed %d\n", __func__, err);
7733         ufshcd_update_evt_hist(hba, UFS_EVT_HOST_RESET, (u32)err);
7734         return err;
7735 }
7736
7737 /**
7738  * ufshcd_reset_and_restore - reset and re-initialize host/device
7739  * @hba: per-adapter instance
7740  *
7741  * Reset and recover device, host and re-establish link. This
7742  * is helpful to recover the communication in fatal error conditions.
7743  *
7744  * Return: zero on success, non-zero on failure.
7745  */
7746 static int ufshcd_reset_and_restore(struct ufs_hba *hba)
7747 {
7748         u32 saved_err = 0;
7749         u32 saved_uic_err = 0;
7750         int err = 0;
7751         unsigned long flags;
7752         int retries = MAX_HOST_RESET_RETRIES;
7753
7754         spin_lock_irqsave(hba->host->host_lock, flags);
7755         do {
7756                 /*
7757                  * This is a fresh start, cache and clear saved error first,
7758                  * in case new error generated during reset and restore.
7759                  */
7760                 saved_err |= hba->saved_err;
7761                 saved_uic_err |= hba->saved_uic_err;
7762                 hba->saved_err = 0;
7763                 hba->saved_uic_err = 0;
7764                 hba->force_reset = false;
7765                 hba->ufshcd_state = UFSHCD_STATE_RESET;
7766                 spin_unlock_irqrestore(hba->host->host_lock, flags);
7767
7768                 /* Reset the attached device */
7769                 ufshcd_device_reset(hba);
7770
7771                 err = ufshcd_host_reset_and_restore(hba);
7772
7773                 spin_lock_irqsave(hba->host->host_lock, flags);
7774                 if (err)
7775                         continue;
7776                 /* Do not exit unless operational or dead */
7777                 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL &&
7778                     hba->ufshcd_state != UFSHCD_STATE_ERROR &&
7779                     hba->ufshcd_state != UFSHCD_STATE_EH_SCHEDULED_NON_FATAL)
7780                         err = -EAGAIN;
7781         } while (err && --retries);
7782
7783         /*
7784          * Inform scsi mid-layer that we did reset and allow to handle
7785          * Unit Attention properly.
7786          */
7787         scsi_report_bus_reset(hba->host, 0);
7788         if (err) {
7789                 hba->ufshcd_state = UFSHCD_STATE_ERROR;
7790                 hba->saved_err |= saved_err;
7791                 hba->saved_uic_err |= saved_uic_err;
7792         }
7793         spin_unlock_irqrestore(hba->host->host_lock, flags);
7794
7795         return err;
7796 }
7797
7798 /**
7799  * ufshcd_eh_host_reset_handler - host reset handler registered to scsi layer
7800  * @cmd: SCSI command pointer
7801  *
7802  * Return: SUCCESS or FAILED.
7803  */
7804 static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd)
7805 {
7806         int err = SUCCESS;
7807         unsigned long flags;
7808         struct ufs_hba *hba;
7809
7810         hba = shost_priv(cmd->device->host);
7811
7812         /*
7813          * If runtime PM sent SSU and got a timeout, scsi_error_handler is
7814          * stuck in this function waiting for flush_work(&hba->eh_work). And
7815          * ufshcd_err_handler(eh_work) is stuck waiting for runtime PM. Do
7816          * ufshcd_link_recovery instead of eh_work to prevent deadlock.
7817          */
7818         if (hba->pm_op_in_progress) {
7819                 if (ufshcd_link_recovery(hba))
7820                         err = FAILED;
7821
7822                 return err;
7823         }
7824
7825         spin_lock_irqsave(hba->host->host_lock, flags);
7826         hba->force_reset = true;
7827         ufshcd_schedule_eh_work(hba);
7828         dev_err(hba->dev, "%s: reset in progress - 1\n", __func__);
7829         spin_unlock_irqrestore(hba->host->host_lock, flags);
7830
7831         flush_work(&hba->eh_work);
7832
7833         spin_lock_irqsave(hba->host->host_lock, flags);
7834         if (hba->ufshcd_state == UFSHCD_STATE_ERROR)
7835                 err = FAILED;
7836         spin_unlock_irqrestore(hba->host->host_lock, flags);
7837
7838         return err;
7839 }
7840
7841 /**
7842  * ufshcd_get_max_icc_level - calculate the ICC level
7843  * @sup_curr_uA: max. current supported by the regulator
7844  * @start_scan: row at the desc table to start scan from
7845  * @buff: power descriptor buffer
7846  *
7847  * Return: calculated max ICC level for specific regulator.
7848  */
7849 static u32 ufshcd_get_max_icc_level(int sup_curr_uA, u32 start_scan,
7850                                     const char *buff)
7851 {
7852         int i;
7853         int curr_uA;
7854         u16 data;
7855         u16 unit;
7856
7857         for (i = start_scan; i >= 0; i--) {
7858                 data = get_unaligned_be16(&buff[2 * i]);
7859                 unit = (data & ATTR_ICC_LVL_UNIT_MASK) >>
7860                                                 ATTR_ICC_LVL_UNIT_OFFSET;
7861                 curr_uA = data & ATTR_ICC_LVL_VALUE_MASK;
7862                 switch (unit) {
7863                 case UFSHCD_NANO_AMP:
7864                         curr_uA = curr_uA / 1000;
7865                         break;
7866                 case UFSHCD_MILI_AMP:
7867                         curr_uA = curr_uA * 1000;
7868                         break;
7869                 case UFSHCD_AMP:
7870                         curr_uA = curr_uA * 1000 * 1000;
7871                         break;
7872                 case UFSHCD_MICRO_AMP:
7873                 default:
7874                         break;
7875                 }
7876                 if (sup_curr_uA >= curr_uA)
7877                         break;
7878         }
7879         if (i < 0) {
7880                 i = 0;
7881                 pr_err("%s: Couldn't find valid icc_level = %d", __func__, i);
7882         }
7883
7884         return (u32)i;
7885 }
7886
7887 /**
7888  * ufshcd_find_max_sup_active_icc_level - calculate the max ICC level
7889  * In case regulators are not initialized we'll return 0
7890  * @hba: per-adapter instance
7891  * @desc_buf: power descriptor buffer to extract ICC levels from.
7892  *
7893  * Return: calculated ICC level.
7894  */
7895 static u32 ufshcd_find_max_sup_active_icc_level(struct ufs_hba *hba,
7896                                                 const u8 *desc_buf)
7897 {
7898         u32 icc_level = 0;
7899
7900         if (!hba->vreg_info.vcc || !hba->vreg_info.vccq ||
7901                                                 !hba->vreg_info.vccq2) {
7902                 /*
7903                  * Using dev_dbg to avoid messages during runtime PM to avoid
7904                  * never-ending cycles of messages written back to storage by
7905                  * user space causing runtime resume, causing more messages and
7906                  * so on.
7907                  */
7908                 dev_dbg(hba->dev,
7909                         "%s: Regulator capability was not set, actvIccLevel=%d",
7910                                                         __func__, icc_level);
7911                 goto out;
7912         }
7913
7914         if (hba->vreg_info.vcc->max_uA)
7915                 icc_level = ufshcd_get_max_icc_level(
7916                                 hba->vreg_info.vcc->max_uA,
7917                                 POWER_DESC_MAX_ACTV_ICC_LVLS - 1,
7918                                 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCC_0]);
7919
7920         if (hba->vreg_info.vccq->max_uA)
7921                 icc_level = ufshcd_get_max_icc_level(
7922                                 hba->vreg_info.vccq->max_uA,
7923                                 icc_level,
7924                                 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ_0]);
7925
7926         if (hba->vreg_info.vccq2->max_uA)
7927                 icc_level = ufshcd_get_max_icc_level(
7928                                 hba->vreg_info.vccq2->max_uA,
7929                                 icc_level,
7930                                 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ2_0]);
7931 out:
7932         return icc_level;
7933 }
7934
7935 static void ufshcd_set_active_icc_lvl(struct ufs_hba *hba)
7936 {
7937         int ret;
7938         u8 *desc_buf;
7939         u32 icc_level;
7940
7941         desc_buf = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL);
7942         if (!desc_buf)
7943                 return;
7944
7945         ret = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_POWER, 0, 0,
7946                                      desc_buf, QUERY_DESC_MAX_SIZE);
7947         if (ret) {
7948                 dev_err(hba->dev,
7949                         "%s: Failed reading power descriptor ret = %d",
7950                         __func__, ret);
7951                 goto out;
7952         }
7953
7954         icc_level = ufshcd_find_max_sup_active_icc_level(hba, desc_buf);
7955         dev_dbg(hba->dev, "%s: setting icc_level 0x%x", __func__, icc_level);
7956
7957         ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
7958                 QUERY_ATTR_IDN_ACTIVE_ICC_LVL, 0, 0, &icc_level);
7959
7960         if (ret)
7961                 dev_err(hba->dev,
7962                         "%s: Failed configuring bActiveICCLevel = %d ret = %d",
7963                         __func__, icc_level, ret);
7964
7965 out:
7966         kfree(desc_buf);
7967 }
7968
7969 static inline void ufshcd_blk_pm_runtime_init(struct scsi_device *sdev)
7970 {
7971         struct Scsi_Host *shost = sdev->host;
7972
7973         scsi_autopm_get_device(sdev);
7974         blk_pm_runtime_init(sdev->request_queue, &sdev->sdev_gendev);
7975         if (sdev->rpm_autosuspend)
7976                 pm_runtime_set_autosuspend_delay(&sdev->sdev_gendev,
7977                                                  shost->rpm_autosuspend_delay);
7978         scsi_autopm_put_device(sdev);
7979 }
7980
7981 /**
7982  * ufshcd_scsi_add_wlus - Adds required W-LUs
7983  * @hba: per-adapter instance
7984  *
7985  * UFS device specification requires the UFS devices to support 4 well known
7986  * logical units:
7987  *      "REPORT_LUNS" (address: 01h)
7988  *      "UFS Device" (address: 50h)
7989  *      "RPMB" (address: 44h)
7990  *      "BOOT" (address: 30h)
7991  * UFS device's power management needs to be controlled by "POWER CONDITION"
7992  * field of SSU (START STOP UNIT) command. But this "power condition" field
7993  * will take effect only when its sent to "UFS device" well known logical unit
7994  * hence we require the scsi_device instance to represent this logical unit in
7995  * order for the UFS host driver to send the SSU command for power management.
7996  *
7997  * We also require the scsi_device instance for "RPMB" (Replay Protected Memory
7998  * Block) LU so user space process can control this LU. User space may also
7999  * want to have access to BOOT LU.
8000  *
8001  * This function adds scsi device instances for each of all well known LUs
8002  * (except "REPORT LUNS" LU).
8003  *
8004  * Return: zero on success (all required W-LUs are added successfully),
8005  * non-zero error value on failure (if failed to add any of the required W-LU).
8006  */
8007 static int ufshcd_scsi_add_wlus(struct ufs_hba *hba)
8008 {
8009         int ret = 0;
8010         struct scsi_device *sdev_boot, *sdev_rpmb;
8011
8012         hba->ufs_device_wlun = __scsi_add_device(hba->host, 0, 0,
8013                 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN), NULL);
8014         if (IS_ERR(hba->ufs_device_wlun)) {
8015                 ret = PTR_ERR(hba->ufs_device_wlun);
8016                 hba->ufs_device_wlun = NULL;
8017                 goto out;
8018         }
8019         scsi_device_put(hba->ufs_device_wlun);
8020
8021         sdev_rpmb = __scsi_add_device(hba->host, 0, 0,
8022                 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_RPMB_WLUN), NULL);
8023         if (IS_ERR(sdev_rpmb)) {
8024                 ret = PTR_ERR(sdev_rpmb);
8025                 goto remove_ufs_device_wlun;
8026         }
8027         ufshcd_blk_pm_runtime_init(sdev_rpmb);
8028         scsi_device_put(sdev_rpmb);
8029
8030         sdev_boot = __scsi_add_device(hba->host, 0, 0,
8031                 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_BOOT_WLUN), NULL);
8032         if (IS_ERR(sdev_boot)) {
8033                 dev_err(hba->dev, "%s: BOOT WLUN not found\n", __func__);
8034         } else {
8035                 ufshcd_blk_pm_runtime_init(sdev_boot);
8036                 scsi_device_put(sdev_boot);
8037         }
8038         goto out;
8039
8040 remove_ufs_device_wlun:
8041         scsi_remove_device(hba->ufs_device_wlun);
8042 out:
8043         return ret;
8044 }
8045
8046 static void ufshcd_wb_probe(struct ufs_hba *hba, const u8 *desc_buf)
8047 {
8048         struct ufs_dev_info *dev_info = &hba->dev_info;
8049         u8 lun;
8050         u32 d_lu_wb_buf_alloc;
8051         u32 ext_ufs_feature;
8052
8053         if (!ufshcd_is_wb_allowed(hba))
8054                 return;
8055
8056         /*
8057          * Probe WB only for UFS-2.2 and UFS-3.1 (and later) devices or
8058          * UFS devices with quirk UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES
8059          * enabled
8060          */
8061         if (!(dev_info->wspecversion >= 0x310 ||
8062               dev_info->wspecversion == 0x220 ||
8063              (hba->dev_quirks & UFS_DEVICE_QUIRK_SUPPORT_EXTENDED_FEATURES)))
8064                 goto wb_disabled;
8065
8066         ext_ufs_feature = get_unaligned_be32(desc_buf +
8067                                         DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP);
8068
8069         if (!(ext_ufs_feature & UFS_DEV_WRITE_BOOSTER_SUP))
8070                 goto wb_disabled;
8071
8072         /*
8073          * WB may be supported but not configured while provisioning. The spec
8074          * says, in dedicated wb buffer mode, a max of 1 lun would have wb
8075          * buffer configured.
8076          */
8077         dev_info->wb_buffer_type = desc_buf[DEVICE_DESC_PARAM_WB_TYPE];
8078
8079         dev_info->b_presrv_uspc_en =
8080                 desc_buf[DEVICE_DESC_PARAM_WB_PRESRV_USRSPC_EN];
8081
8082         if (dev_info->wb_buffer_type == WB_BUF_MODE_SHARED) {
8083                 if (!get_unaligned_be32(desc_buf +
8084                                    DEVICE_DESC_PARAM_WB_SHARED_ALLOC_UNITS))
8085                         goto wb_disabled;
8086         } else {
8087                 for (lun = 0; lun < UFS_UPIU_MAX_WB_LUN_ID; lun++) {
8088                         d_lu_wb_buf_alloc = 0;
8089                         ufshcd_read_unit_desc_param(hba,
8090                                         lun,
8091                                         UNIT_DESC_PARAM_WB_BUF_ALLOC_UNITS,
8092                                         (u8 *)&d_lu_wb_buf_alloc,
8093                                         sizeof(d_lu_wb_buf_alloc));
8094                         if (d_lu_wb_buf_alloc) {
8095                                 dev_info->wb_dedicated_lu = lun;
8096                                 break;
8097                         }
8098                 }
8099
8100                 if (!d_lu_wb_buf_alloc)
8101                         goto wb_disabled;
8102         }
8103
8104         if (!ufshcd_is_wb_buf_lifetime_available(hba))
8105                 goto wb_disabled;
8106
8107         return;
8108
8109 wb_disabled:
8110         hba->caps &= ~UFSHCD_CAP_WB_EN;
8111 }
8112
8113 static void ufshcd_temp_notif_probe(struct ufs_hba *hba, const u8 *desc_buf)
8114 {
8115         struct ufs_dev_info *dev_info = &hba->dev_info;
8116         u32 ext_ufs_feature;
8117         u8 mask = 0;
8118
8119         if (!(hba->caps & UFSHCD_CAP_TEMP_NOTIF) || dev_info->wspecversion < 0x300)
8120                 return;
8121
8122         ext_ufs_feature = get_unaligned_be32(desc_buf + DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP);
8123
8124         if (ext_ufs_feature & UFS_DEV_LOW_TEMP_NOTIF)
8125                 mask |= MASK_EE_TOO_LOW_TEMP;
8126
8127         if (ext_ufs_feature & UFS_DEV_HIGH_TEMP_NOTIF)
8128                 mask |= MASK_EE_TOO_HIGH_TEMP;
8129
8130         if (mask) {
8131                 ufshcd_enable_ee(hba, mask);
8132                 ufs_hwmon_probe(hba, mask);
8133         }
8134 }
8135
8136 static void ufshcd_ext_iid_probe(struct ufs_hba *hba, u8 *desc_buf)
8137 {
8138         struct ufs_dev_info *dev_info = &hba->dev_info;
8139         u32 ext_ufs_feature;
8140         u32 ext_iid_en = 0;
8141         int err;
8142
8143         /* Only UFS-4.0 and above may support EXT_IID */
8144         if (dev_info->wspecversion < 0x400)
8145                 goto out;
8146
8147         ext_ufs_feature = get_unaligned_be32(desc_buf +
8148                                      DEVICE_DESC_PARAM_EXT_UFS_FEATURE_SUP);
8149         if (!(ext_ufs_feature & UFS_DEV_EXT_IID_SUP))
8150                 goto out;
8151
8152         err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
8153                                       QUERY_ATTR_IDN_EXT_IID_EN, 0, 0, &ext_iid_en);
8154         if (err)
8155                 dev_err(hba->dev, "failed reading bEXTIIDEn. err = %d\n", err);
8156
8157 out:
8158         dev_info->b_ext_iid_en = ext_iid_en;
8159 }
8160
8161 static void ufshcd_set_rtt(struct ufs_hba *hba)
8162 {
8163         struct ufs_dev_info *dev_info = &hba->dev_info;
8164         u32 rtt = 0;
8165         u32 dev_rtt = 0;
8166         int host_rtt_cap = hba->vops && hba->vops->max_num_rtt ?
8167                            hba->vops->max_num_rtt : hba->nortt;
8168
8169         /* RTT override makes sense only for UFS-4.0 and above */
8170         if (dev_info->wspecversion < 0x400)
8171                 return;
8172
8173         if (ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
8174                                     QUERY_ATTR_IDN_MAX_NUM_OF_RTT, 0, 0, &dev_rtt)) {
8175                 dev_err(hba->dev, "failed reading bMaxNumOfRTT\n");
8176                 return;
8177         }
8178
8179         /* do not override if it was already written */
8180         if (dev_rtt != DEFAULT_MAX_NUM_RTT)
8181                 return;
8182
8183         rtt = min_t(int, dev_info->rtt_cap, host_rtt_cap);
8184
8185         if (rtt == dev_rtt)
8186                 return;
8187
8188         if (ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
8189                                     QUERY_ATTR_IDN_MAX_NUM_OF_RTT, 0, 0, &rtt))
8190                 dev_err(hba->dev, "failed writing bMaxNumOfRTT\n");
8191 }
8192
8193 void ufshcd_fixup_dev_quirks(struct ufs_hba *hba,
8194                              const struct ufs_dev_quirk *fixups)
8195 {
8196         const struct ufs_dev_quirk *f;
8197         struct ufs_dev_info *dev_info = &hba->dev_info;
8198
8199         if (!fixups)
8200                 return;
8201
8202         for (f = fixups; f->quirk; f++) {
8203                 if ((f->wmanufacturerid == dev_info->wmanufacturerid ||
8204                      f->wmanufacturerid == UFS_ANY_VENDOR) &&
8205                      ((dev_info->model &&
8206                        STR_PRFX_EQUAL(f->model, dev_info->model)) ||
8207                       !strcmp(f->model, UFS_ANY_MODEL)))
8208                         hba->dev_quirks |= f->quirk;
8209         }
8210 }
8211 EXPORT_SYMBOL_GPL(ufshcd_fixup_dev_quirks);
8212
8213 static void ufs_fixup_device_setup(struct ufs_hba *hba)
8214 {
8215         /* fix by general quirk table */
8216         ufshcd_fixup_dev_quirks(hba, ufs_fixups);
8217
8218         /* allow vendors to fix quirks */
8219         ufshcd_vops_fixup_dev_quirks(hba);
8220 }
8221
8222 static void ufshcd_update_rtc(struct ufs_hba *hba)
8223 {
8224         struct timespec64 ts64;
8225         int err;
8226         u32 val;
8227
8228         ktime_get_real_ts64(&ts64);
8229
8230         if (ts64.tv_sec < hba->dev_info.rtc_time_baseline) {
8231                 dev_warn_once(hba->dev, "%s: Current time precedes previous setting!\n", __func__);
8232                 return;
8233         }
8234
8235         /*
8236          * The Absolute RTC mode has a 136-year limit, spanning from 2010 to 2146. If a time beyond
8237          * 2146 is required, it is recommended to choose the relative RTC mode.
8238          */
8239         val = ts64.tv_sec - hba->dev_info.rtc_time_baseline;
8240
8241         /* Skip update RTC if RPM state is not RPM_ACTIVE */
8242         if (ufshcd_rpm_get_if_active(hba) <= 0)
8243                 return;
8244
8245         err = ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_WRITE_ATTR, QUERY_ATTR_IDN_SECONDS_PASSED,
8246                                 0, 0, &val);
8247         ufshcd_rpm_put(hba);
8248
8249         if (err)
8250                 dev_err(hba->dev, "%s: Failed to update rtc %d\n", __func__, err);
8251         else if (hba->dev_info.rtc_type == UFS_RTC_RELATIVE)
8252                 hba->dev_info.rtc_time_baseline = ts64.tv_sec;
8253 }
8254
8255 static void ufshcd_rtc_work(struct work_struct *work)
8256 {
8257         struct ufs_hba *hba;
8258
8259         hba = container_of(to_delayed_work(work), struct ufs_hba, ufs_rtc_update_work);
8260
8261          /* Update RTC only when there are no requests in progress and UFSHCI is operational */
8262         if (!ufshcd_is_ufs_dev_busy(hba) && hba->ufshcd_state == UFSHCD_STATE_OPERATIONAL)
8263                 ufshcd_update_rtc(hba);
8264
8265         if (ufshcd_is_ufs_dev_active(hba) && hba->dev_info.rtc_update_period)
8266                 schedule_delayed_work(&hba->ufs_rtc_update_work,
8267                                       msecs_to_jiffies(hba->dev_info.rtc_update_period));
8268 }
8269
8270 static void ufs_init_rtc(struct ufs_hba *hba, u8 *desc_buf)
8271 {
8272         u16 periodic_rtc_update = get_unaligned_be16(&desc_buf[DEVICE_DESC_PARAM_FRQ_RTC]);
8273         struct ufs_dev_info *dev_info = &hba->dev_info;
8274
8275         if (periodic_rtc_update & UFS_RTC_TIME_BASELINE) {
8276                 dev_info->rtc_type = UFS_RTC_ABSOLUTE;
8277
8278                 /*
8279                  * The concept of measuring time in Linux as the number of seconds elapsed since
8280                  * 00:00:00 UTC on January 1, 1970, and UFS ABS RTC is elapsed from January 1st
8281                  * 2010 00:00, here we need to adjust ABS baseline.
8282                  */
8283                 dev_info->rtc_time_baseline = mktime64(2010, 1, 1, 0, 0, 0) -
8284                                                         mktime64(1970, 1, 1, 0, 0, 0);
8285         } else {
8286                 dev_info->rtc_type = UFS_RTC_RELATIVE;
8287                 dev_info->rtc_time_baseline = 0;
8288         }
8289
8290         /*
8291          * We ignore TIME_PERIOD defined in wPeriodicRTCUpdate because Spec does not clearly state
8292          * how to calculate the specific update period for each time unit. And we disable periodic
8293          * RTC update work, let user configure by sysfs node according to specific circumstance.
8294          */
8295         dev_info->rtc_update_period = 0;
8296 }
8297
8298 static int ufs_get_device_desc(struct ufs_hba *hba)
8299 {
8300         int err;
8301         u8 model_index;
8302         u8 *desc_buf;
8303         struct ufs_dev_info *dev_info = &hba->dev_info;
8304
8305         desc_buf = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL);
8306         if (!desc_buf) {
8307                 err = -ENOMEM;
8308                 goto out;
8309         }
8310
8311         err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_DEVICE, 0, 0, desc_buf,
8312                                      QUERY_DESC_MAX_SIZE);
8313         if (err) {
8314                 dev_err(hba->dev, "%s: Failed reading Device Desc. err = %d\n",
8315                         __func__, err);
8316                 goto out;
8317         }
8318
8319         /*
8320          * getting vendor (manufacturerID) and Bank Index in big endian
8321          * format
8322          */
8323         dev_info->wmanufacturerid = desc_buf[DEVICE_DESC_PARAM_MANF_ID] << 8 |
8324                                      desc_buf[DEVICE_DESC_PARAM_MANF_ID + 1];
8325
8326         /* getting Specification Version in big endian format */
8327         dev_info->wspecversion = desc_buf[DEVICE_DESC_PARAM_SPEC_VER] << 8 |
8328                                       desc_buf[DEVICE_DESC_PARAM_SPEC_VER + 1];
8329         dev_info->bqueuedepth = desc_buf[DEVICE_DESC_PARAM_Q_DPTH];
8330
8331         dev_info->rtt_cap = desc_buf[DEVICE_DESC_PARAM_RTT_CAP];
8332
8333         model_index = desc_buf[DEVICE_DESC_PARAM_PRDCT_NAME];
8334
8335         err = ufshcd_read_string_desc(hba, model_index,
8336                                       &dev_info->model, SD_ASCII_STD);
8337         if (err < 0) {
8338                 dev_err(hba->dev, "%s: Failed reading Product Name. err = %d\n",
8339                         __func__, err);
8340                 goto out;
8341         }
8342
8343         hba->luns_avail = desc_buf[DEVICE_DESC_PARAM_NUM_LU] +
8344                 desc_buf[DEVICE_DESC_PARAM_NUM_WLU];
8345
8346         ufs_fixup_device_setup(hba);
8347
8348         ufshcd_wb_probe(hba, desc_buf);
8349
8350         ufshcd_temp_notif_probe(hba, desc_buf);
8351
8352         ufs_init_rtc(hba, desc_buf);
8353
8354         if (hba->ext_iid_sup)
8355                 ufshcd_ext_iid_probe(hba, desc_buf);
8356
8357         /*
8358          * ufshcd_read_string_desc returns size of the string
8359          * reset the error value
8360          */
8361         err = 0;
8362
8363 out:
8364         kfree(desc_buf);
8365         return err;
8366 }
8367
8368 static void ufs_put_device_desc(struct ufs_hba *hba)
8369 {
8370         struct ufs_dev_info *dev_info = &hba->dev_info;
8371
8372         kfree(dev_info->model);
8373         dev_info->model = NULL;
8374 }
8375
8376 /**
8377  * ufshcd_quirk_tune_host_pa_tactivate - Ensures that host PA_TACTIVATE is
8378  * less than device PA_TACTIVATE time.
8379  * @hba: per-adapter instance
8380  *
8381  * Some UFS devices require host PA_TACTIVATE to be lower than device
8382  * PA_TACTIVATE, we need to enable UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE quirk
8383  * for such devices.
8384  *
8385  * Return: zero on success, non-zero error value on failure.
8386  */
8387 static int ufshcd_quirk_tune_host_pa_tactivate(struct ufs_hba *hba)
8388 {
8389         int ret = 0;
8390         u32 granularity, peer_granularity;
8391         u32 pa_tactivate, peer_pa_tactivate;
8392         u32 pa_tactivate_us, peer_pa_tactivate_us;
8393         static const u8 gran_to_us_table[] = {1, 4, 8, 16, 32, 100};
8394
8395         ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
8396                                   &granularity);
8397         if (ret)
8398                 goto out;
8399
8400         ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
8401                                   &peer_granularity);
8402         if (ret)
8403                 goto out;
8404
8405         if ((granularity < PA_GRANULARITY_MIN_VAL) ||
8406             (granularity > PA_GRANULARITY_MAX_VAL)) {
8407                 dev_err(hba->dev, "%s: invalid host PA_GRANULARITY %d",
8408                         __func__, granularity);
8409                 return -EINVAL;
8410         }
8411
8412         if ((peer_granularity < PA_GRANULARITY_MIN_VAL) ||
8413             (peer_granularity > PA_GRANULARITY_MAX_VAL)) {
8414                 dev_err(hba->dev, "%s: invalid device PA_GRANULARITY %d",
8415                         __func__, peer_granularity);
8416                 return -EINVAL;
8417         }
8418
8419         ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_TACTIVATE), &pa_tactivate);
8420         if (ret)
8421                 goto out;
8422
8423         ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_TACTIVATE),
8424                                   &peer_pa_tactivate);
8425         if (ret)
8426                 goto out;
8427
8428         pa_tactivate_us = pa_tactivate * gran_to_us_table[granularity - 1];
8429         peer_pa_tactivate_us = peer_pa_tactivate *
8430                              gran_to_us_table[peer_granularity - 1];
8431
8432         if (pa_tactivate_us >= peer_pa_tactivate_us) {
8433                 u32 new_peer_pa_tactivate;
8434
8435                 new_peer_pa_tactivate = pa_tactivate_us /
8436                                       gran_to_us_table[peer_granularity - 1];
8437                 new_peer_pa_tactivate++;
8438                 ret = ufshcd_dme_peer_set(hba, UIC_ARG_MIB(PA_TACTIVATE),
8439                                           new_peer_pa_tactivate);
8440         }
8441
8442 out:
8443         return ret;
8444 }
8445
8446 static void ufshcd_tune_unipro_params(struct ufs_hba *hba)
8447 {
8448         ufshcd_vops_apply_dev_quirks(hba);
8449
8450         if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_TACTIVATE)
8451                 /* set 1ms timeout for PA_TACTIVATE */
8452                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE), 10);
8453
8454         if (hba->dev_quirks & UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE)
8455                 ufshcd_quirk_tune_host_pa_tactivate(hba);
8456 }
8457
8458 static void ufshcd_clear_dbg_ufs_stats(struct ufs_hba *hba)
8459 {
8460         hba->ufs_stats.hibern8_exit_cnt = 0;
8461         hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
8462         hba->req_abort_count = 0;
8463 }
8464
8465 static int ufshcd_device_geo_params_init(struct ufs_hba *hba)
8466 {
8467         int err;
8468         u8 *desc_buf;
8469
8470         desc_buf = kzalloc(QUERY_DESC_MAX_SIZE, GFP_KERNEL);
8471         if (!desc_buf) {
8472                 err = -ENOMEM;
8473                 goto out;
8474         }
8475
8476         err = ufshcd_read_desc_param(hba, QUERY_DESC_IDN_GEOMETRY, 0, 0,
8477                                      desc_buf, QUERY_DESC_MAX_SIZE);
8478         if (err) {
8479                 dev_err(hba->dev, "%s: Failed reading Geometry Desc. err = %d\n",
8480                                 __func__, err);
8481                 goto out;
8482         }
8483
8484         if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 1)
8485                 hba->dev_info.max_lu_supported = 32;
8486         else if (desc_buf[GEOMETRY_DESC_PARAM_MAX_NUM_LUN] == 0)
8487                 hba->dev_info.max_lu_supported = 8;
8488
8489 out:
8490         kfree(desc_buf);
8491         return err;
8492 }
8493
8494 struct ufs_ref_clk {
8495         unsigned long freq_hz;
8496         enum ufs_ref_clk_freq val;
8497 };
8498
8499 static const struct ufs_ref_clk ufs_ref_clk_freqs[] = {
8500         {19200000, REF_CLK_FREQ_19_2_MHZ},
8501         {26000000, REF_CLK_FREQ_26_MHZ},
8502         {38400000, REF_CLK_FREQ_38_4_MHZ},
8503         {52000000, REF_CLK_FREQ_52_MHZ},
8504         {0, REF_CLK_FREQ_INVAL},
8505 };
8506
8507 static enum ufs_ref_clk_freq
8508 ufs_get_bref_clk_from_hz(unsigned long freq)
8509 {
8510         int i;
8511
8512         for (i = 0; ufs_ref_clk_freqs[i].freq_hz; i++)
8513                 if (ufs_ref_clk_freqs[i].freq_hz == freq)
8514                         return ufs_ref_clk_freqs[i].val;
8515
8516         return REF_CLK_FREQ_INVAL;
8517 }
8518
8519 void ufshcd_parse_dev_ref_clk_freq(struct ufs_hba *hba, struct clk *refclk)
8520 {
8521         unsigned long freq;
8522
8523         freq = clk_get_rate(refclk);
8524
8525         hba->dev_ref_clk_freq =
8526                 ufs_get_bref_clk_from_hz(freq);
8527
8528         if (hba->dev_ref_clk_freq == REF_CLK_FREQ_INVAL)
8529                 dev_err(hba->dev,
8530                 "invalid ref_clk setting = %ld\n", freq);
8531 }
8532
8533 static int ufshcd_set_dev_ref_clk(struct ufs_hba *hba)
8534 {
8535         int err;
8536         u32 ref_clk;
8537         u32 freq = hba->dev_ref_clk_freq;
8538
8539         err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
8540                         QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &ref_clk);
8541
8542         if (err) {
8543                 dev_err(hba->dev, "failed reading bRefClkFreq. err = %d\n",
8544                         err);
8545                 goto out;
8546         }
8547
8548         if (ref_clk == freq)
8549                 goto out; /* nothing to update */
8550
8551         err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
8552                         QUERY_ATTR_IDN_REF_CLK_FREQ, 0, 0, &freq);
8553
8554         if (err) {
8555                 dev_err(hba->dev, "bRefClkFreq setting to %lu Hz failed\n",
8556                         ufs_ref_clk_freqs[freq].freq_hz);
8557                 goto out;
8558         }
8559
8560         dev_dbg(hba->dev, "bRefClkFreq setting to %lu Hz succeeded\n",
8561                         ufs_ref_clk_freqs[freq].freq_hz);
8562
8563 out:
8564         return err;
8565 }
8566
8567 static int ufshcd_device_params_init(struct ufs_hba *hba)
8568 {
8569         bool flag;
8570         int ret;
8571
8572         /* Init UFS geometry descriptor related parameters */
8573         ret = ufshcd_device_geo_params_init(hba);
8574         if (ret)
8575                 goto out;
8576
8577         /* Check and apply UFS device quirks */
8578         ret = ufs_get_device_desc(hba);
8579         if (ret) {
8580                 dev_err(hba->dev, "%s: Failed getting device info. err = %d\n",
8581                         __func__, ret);
8582                 goto out;
8583         }
8584
8585         ufshcd_set_rtt(hba);
8586
8587         ufshcd_get_ref_clk_gating_wait(hba);
8588
8589         if (!ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG,
8590                         QUERY_FLAG_IDN_PWR_ON_WPE, 0, &flag))
8591                 hba->dev_info.f_power_on_wp_en = flag;
8592
8593         /* Probe maximum power mode co-supported by both UFS host and device */
8594         if (ufshcd_get_max_pwr_mode(hba))
8595                 dev_err(hba->dev,
8596                         "%s: Failed getting max supported power mode\n",
8597                         __func__);
8598 out:
8599         return ret;
8600 }
8601
8602 static void ufshcd_set_timestamp_attr(struct ufs_hba *hba)
8603 {
8604         int err;
8605         struct ufs_query_req *request = NULL;
8606         struct ufs_query_res *response = NULL;
8607         struct ufs_dev_info *dev_info = &hba->dev_info;
8608         struct utp_upiu_query_v4_0 *upiu_data;
8609
8610         if (dev_info->wspecversion < 0x400)
8611                 return;
8612
8613         ufshcd_dev_man_lock(hba);
8614
8615         ufshcd_init_query(hba, &request, &response,
8616                           UPIU_QUERY_OPCODE_WRITE_ATTR,
8617                           QUERY_ATTR_IDN_TIMESTAMP, 0, 0);
8618
8619         request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
8620
8621         upiu_data = (struct utp_upiu_query_v4_0 *)&request->upiu_req;
8622
8623         put_unaligned_be64(ktime_get_real_ns(), &upiu_data->osf3);
8624
8625         err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
8626
8627         if (err)
8628                 dev_err(hba->dev, "%s: failed to set timestamp %d\n",
8629                         __func__, err);
8630
8631         ufshcd_dev_man_unlock(hba);
8632 }
8633
8634 /**
8635  * ufshcd_add_lus - probe and add UFS logical units
8636  * @hba: per-adapter instance
8637  *
8638  * Return: 0 upon success; < 0 upon failure.
8639  */
8640 static int ufshcd_add_lus(struct ufs_hba *hba)
8641 {
8642         int ret;
8643
8644         /* Add required well known logical units to scsi mid layer */
8645         ret = ufshcd_scsi_add_wlus(hba);
8646         if (ret)
8647                 goto out;
8648
8649         /* Initialize devfreq after UFS device is detected */
8650         if (ufshcd_is_clkscaling_supported(hba)) {
8651                 memcpy(&hba->clk_scaling.saved_pwr_info,
8652                         &hba->pwr_info,
8653                         sizeof(struct ufs_pa_layer_attr));
8654                 hba->clk_scaling.is_allowed = true;
8655
8656                 ret = ufshcd_devfreq_init(hba);
8657                 if (ret)
8658                         goto out;
8659
8660                 hba->clk_scaling.is_enabled = true;
8661                 ufshcd_init_clk_scaling_sysfs(hba);
8662         }
8663
8664         /*
8665          * The RTC update code accesses the hba->ufs_device_wlun->sdev_gendev
8666          * pointer and hence must only be started after the WLUN pointer has
8667          * been initialized by ufshcd_scsi_add_wlus().
8668          */
8669         schedule_delayed_work(&hba->ufs_rtc_update_work,
8670                               msecs_to_jiffies(UFS_RTC_UPDATE_INTERVAL_MS));
8671
8672         ufs_bsg_probe(hba);
8673         scsi_scan_host(hba->host);
8674
8675 out:
8676         return ret;
8677 }
8678
8679 /* SDB - Single Doorbell */
8680 static void ufshcd_release_sdb_queue(struct ufs_hba *hba, int nutrs)
8681 {
8682         size_t ucdl_size, utrdl_size;
8683
8684         ucdl_size = ufshcd_get_ucd_size(hba) * nutrs;
8685         dmam_free_coherent(hba->dev, ucdl_size, hba->ucdl_base_addr,
8686                            hba->ucdl_dma_addr);
8687
8688         utrdl_size = sizeof(struct utp_transfer_req_desc) * nutrs;
8689         dmam_free_coherent(hba->dev, utrdl_size, hba->utrdl_base_addr,
8690                            hba->utrdl_dma_addr);
8691
8692         devm_kfree(hba->dev, hba->lrb);
8693 }
8694
8695 static int ufshcd_alloc_mcq(struct ufs_hba *hba)
8696 {
8697         int ret;
8698         int old_nutrs = hba->nutrs;
8699
8700         ret = ufshcd_mcq_decide_queue_depth(hba);
8701         if (ret < 0)
8702                 return ret;
8703
8704         hba->nutrs = ret;
8705         ret = ufshcd_mcq_init(hba);
8706         if (ret)
8707                 goto err;
8708
8709         /*
8710          * Previously allocated memory for nutrs may not be enough in MCQ mode.
8711          * Number of supported tags in MCQ mode may be larger than SDB mode.
8712          */
8713         if (hba->nutrs != old_nutrs) {
8714                 ufshcd_release_sdb_queue(hba, old_nutrs);
8715                 ret = ufshcd_memory_alloc(hba);
8716                 if (ret)
8717                         goto err;
8718                 ufshcd_host_memory_configure(hba);
8719         }
8720
8721         ret = ufshcd_mcq_memory_alloc(hba);
8722         if (ret)
8723                 goto err;
8724
8725         hba->host->can_queue = hba->nutrs - UFSHCD_NUM_RESERVED;
8726         hba->reserved_slot = hba->nutrs - UFSHCD_NUM_RESERVED;
8727
8728         return 0;
8729 err:
8730         hba->nutrs = old_nutrs;
8731         return ret;
8732 }
8733
8734 static void ufshcd_config_mcq(struct ufs_hba *hba)
8735 {
8736         int ret;
8737         u32 intrs;
8738
8739         ret = ufshcd_mcq_vops_config_esi(hba);
8740         dev_info(hba->dev, "ESI %sconfigured\n", ret ? "is not " : "");
8741
8742         intrs = UFSHCD_ENABLE_MCQ_INTRS;
8743         if (hba->quirks & UFSHCD_QUIRK_MCQ_BROKEN_INTR)
8744                 intrs &= ~MCQ_CQ_EVENT_STATUS;
8745         ufshcd_enable_intr(hba, intrs);
8746         ufshcd_mcq_make_queues_operational(hba);
8747         ufshcd_mcq_config_mac(hba, hba->nutrs);
8748
8749         dev_info(hba->dev, "MCQ configured, nr_queues=%d, io_queues=%d, read_queue=%d, poll_queues=%d, queue_depth=%d\n",
8750                  hba->nr_hw_queues, hba->nr_queues[HCTX_TYPE_DEFAULT],
8751                  hba->nr_queues[HCTX_TYPE_READ], hba->nr_queues[HCTX_TYPE_POLL],
8752                  hba->nutrs);
8753 }
8754
8755 static int ufshcd_post_device_init(struct ufs_hba *hba)
8756 {
8757         int ret;
8758
8759         ufshcd_tune_unipro_params(hba);
8760
8761         /* UFS device is also active now */
8762         ufshcd_set_ufs_dev_active(hba);
8763         ufshcd_force_reset_auto_bkops(hba);
8764
8765         ufshcd_set_timestamp_attr(hba);
8766
8767         if (!hba->max_pwr_info.is_valid)
8768                 return 0;
8769
8770         /*
8771          * Set the right value to bRefClkFreq before attempting to
8772          * switch to HS gears.
8773          */
8774         if (hba->dev_ref_clk_freq != REF_CLK_FREQ_INVAL)
8775                 ufshcd_set_dev_ref_clk(hba);
8776         /* Gear up to HS gear. */
8777         ret = ufshcd_config_pwr_mode(hba, &hba->max_pwr_info.info);
8778         if (ret) {
8779                 dev_err(hba->dev, "%s: Failed setting power mode, err = %d\n",
8780                         __func__, ret);
8781                 return ret;
8782         }
8783
8784         return 0;
8785 }
8786
8787 static int ufshcd_device_init(struct ufs_hba *hba, bool init_dev_params)
8788 {
8789         int ret;
8790
8791         WARN_ON_ONCE(!hba->scsi_host_added);
8792
8793         hba->ufshcd_state = UFSHCD_STATE_RESET;
8794
8795         ret = ufshcd_link_startup(hba);
8796         if (ret)
8797                 return ret;
8798
8799         if (hba->quirks & UFSHCD_QUIRK_SKIP_PH_CONFIGURATION)
8800                 return ret;
8801
8802         /* Debug counters initialization */
8803         ufshcd_clear_dbg_ufs_stats(hba);
8804
8805         /* UniPro link is active now */
8806         ufshcd_set_link_active(hba);
8807
8808         /* Reconfigure MCQ upon reset */
8809         if (hba->mcq_enabled && !init_dev_params) {
8810                 ufshcd_config_mcq(hba);
8811                 ufshcd_mcq_enable(hba);
8812         }
8813
8814         /* Verify device initialization by sending NOP OUT UPIU */
8815         ret = ufshcd_verify_dev_init(hba);
8816         if (ret)
8817                 return ret;
8818
8819         /* Initiate UFS initialization, and waiting until completion */
8820         ret = ufshcd_complete_dev_init(hba);
8821         if (ret)
8822                 return ret;
8823
8824         /*
8825          * Initialize UFS device parameters used by driver, these
8826          * parameters are associated with UFS descriptors.
8827          */
8828         if (init_dev_params) {
8829                 ret = ufshcd_device_params_init(hba);
8830                 if (ret)
8831                         return ret;
8832                 if (is_mcq_supported(hba) &&
8833                     hba->quirks & UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH) {
8834                         ufshcd_config_mcq(hba);
8835                         ufshcd_mcq_enable(hba);
8836                 }
8837         }
8838
8839         return ufshcd_post_device_init(hba);
8840 }
8841
8842 /**
8843  * ufshcd_probe_hba - probe hba to detect device and initialize it
8844  * @hba: per-adapter instance
8845  * @init_dev_params: whether or not to call ufshcd_device_params_init().
8846  *
8847  * Execute link-startup and verify device initialization
8848  *
8849  * Return: 0 upon success; < 0 upon failure.
8850  */
8851 static int ufshcd_probe_hba(struct ufs_hba *hba, bool init_dev_params)
8852 {
8853         int ret;
8854
8855         if (!hba->pm_op_in_progress &&
8856             (hba->quirks & UFSHCD_QUIRK_REINIT_AFTER_MAX_GEAR_SWITCH)) {
8857                 /* Reset the device and controller before doing reinit */
8858                 ufshcd_device_reset(hba);
8859                 ufs_put_device_desc(hba);
8860                 ufshcd_hba_stop(hba);
8861                 ufshcd_vops_reinit_notify(hba);
8862                 ret = ufshcd_hba_enable(hba);
8863                 if (ret) {
8864                         dev_err(hba->dev, "Host controller enable failed\n");
8865                         ufshcd_print_evt_hist(hba);
8866                         ufshcd_print_host_state(hba);
8867                         return ret;
8868                 }
8869
8870                 /* Reinit the device */
8871                 ret = ufshcd_device_init(hba, init_dev_params);
8872                 if (ret)
8873                         return ret;
8874         }
8875
8876         ufshcd_print_pwr_info(hba);
8877
8878         /*
8879          * bActiveICCLevel is volatile for UFS device (as per latest v2.1 spec)
8880          * and for removable UFS card as well, hence always set the parameter.
8881          * Note: Error handler may issue the device reset hence resetting
8882          * bActiveICCLevel as well so it is always safe to set this here.
8883          */
8884         ufshcd_set_active_icc_lvl(hba);
8885
8886         /* Enable UFS Write Booster if supported */
8887         ufshcd_configure_wb(hba);
8888
8889         if (hba->ee_usr_mask)
8890                 ufshcd_write_ee_control(hba);
8891         ufshcd_configure_auto_hibern8(hba);
8892
8893         return 0;
8894 }
8895
8896 /**
8897  * ufshcd_async_scan - asynchronous execution for probing hba
8898  * @data: data pointer to pass to this function
8899  * @cookie: cookie data
8900  */
8901 static void ufshcd_async_scan(void *data, async_cookie_t cookie)
8902 {
8903         struct ufs_hba *hba = (struct ufs_hba *)data;
8904         ktime_t probe_start;
8905         int ret;
8906
8907         down(&hba->host_sem);
8908         /* Initialize hba, detect and initialize UFS device */
8909         probe_start = ktime_get();
8910         ret = ufshcd_probe_hba(hba, true);
8911         ufshcd_process_probe_result(hba, probe_start, ret);
8912         up(&hba->host_sem);
8913         if (ret)
8914                 goto out;
8915
8916         /* Probe and add UFS logical units  */
8917         ret = ufshcd_add_lus(hba);
8918
8919 out:
8920         pm_runtime_put_sync(hba->dev);
8921
8922         if (ret)
8923                 dev_err(hba->dev, "%s failed: %d\n", __func__, ret);
8924 }
8925
8926 static enum scsi_timeout_action ufshcd_eh_timed_out(struct scsi_cmnd *scmd)
8927 {
8928         struct ufs_hba *hba = shost_priv(scmd->device->host);
8929
8930         if (!hba->system_suspending) {
8931                 /* Activate the error handler in the SCSI core. */
8932                 return SCSI_EH_NOT_HANDLED;
8933         }
8934
8935         /*
8936          * If we get here we know that no TMFs are outstanding and also that
8937          * the only pending command is a START STOP UNIT command. Handle the
8938          * timeout of that command directly to prevent a deadlock between
8939          * ufshcd_set_dev_pwr_mode() and ufshcd_err_handler().
8940          */
8941         ufshcd_link_recovery(hba);
8942         dev_info(hba->dev, "%s() finished; outstanding_tasks = %#lx.\n",
8943                  __func__, hba->outstanding_tasks);
8944
8945         return hba->outstanding_reqs ? SCSI_EH_RESET_TIMER : SCSI_EH_DONE;
8946 }
8947
8948 static const struct attribute_group *ufshcd_driver_groups[] = {
8949         &ufs_sysfs_unit_descriptor_group,
8950         &ufs_sysfs_lun_attributes_group,
8951         NULL,
8952 };
8953
8954 static struct ufs_hba_variant_params ufs_hba_vps = {
8955         .hba_enable_delay_us            = 1000,
8956         .wb_flush_threshold             = UFS_WB_BUF_REMAIN_PERCENT(40),
8957         .devfreq_profile.polling_ms     = 100,
8958         .devfreq_profile.target         = ufshcd_devfreq_target,
8959         .devfreq_profile.get_dev_status = ufshcd_devfreq_get_dev_status,
8960         .ondemand_data.upthreshold      = 70,
8961         .ondemand_data.downdifferential = 5,
8962 };
8963
8964 static const struct scsi_host_template ufshcd_driver_template = {
8965         .module                 = THIS_MODULE,
8966         .name                   = UFSHCD,
8967         .proc_name              = UFSHCD,
8968         .map_queues             = ufshcd_map_queues,
8969         .queuecommand           = ufshcd_queuecommand,
8970         .mq_poll                = ufshcd_poll,
8971         .slave_alloc            = ufshcd_slave_alloc,
8972         .device_configure       = ufshcd_device_configure,
8973         .slave_destroy          = ufshcd_slave_destroy,
8974         .change_queue_depth     = ufshcd_change_queue_depth,
8975         .eh_abort_handler       = ufshcd_abort,
8976         .eh_device_reset_handler = ufshcd_eh_device_reset_handler,
8977         .eh_host_reset_handler   = ufshcd_eh_host_reset_handler,
8978         .eh_timed_out           = ufshcd_eh_timed_out,
8979         .this_id                = -1,
8980         .sg_tablesize           = SG_ALL,
8981         .max_segment_size       = PRDT_DATA_BYTE_COUNT_MAX,
8982         .max_sectors            = SZ_1M / SECTOR_SIZE,
8983         .max_host_blocked       = 1,
8984         .track_queue_depth      = 1,
8985         .skip_settle_delay      = 1,
8986         .sdev_groups            = ufshcd_driver_groups,
8987 };
8988
8989 static int ufshcd_config_vreg_load(struct device *dev, struct ufs_vreg *vreg,
8990                                    int ua)
8991 {
8992         int ret;
8993
8994         if (!vreg)
8995                 return 0;
8996
8997         /*
8998          * "set_load" operation shall be required on those regulators
8999          * which specifically configured current limitation. Otherwise
9000          * zero max_uA may cause unexpected behavior when regulator is
9001          * enabled or set as high power mode.
9002          */
9003         if (!vreg->max_uA)
9004                 return 0;
9005
9006         ret = regulator_set_load(vreg->reg, ua);
9007         if (ret < 0) {
9008                 dev_err(dev, "%s: %s set load (ua=%d) failed, err=%d\n",
9009                                 __func__, vreg->name, ua, ret);
9010         }
9011
9012         return ret;
9013 }
9014
9015 static inline int ufshcd_config_vreg_lpm(struct ufs_hba *hba,
9016                                          struct ufs_vreg *vreg)
9017 {
9018         return ufshcd_config_vreg_load(hba->dev, vreg, UFS_VREG_LPM_LOAD_UA);
9019 }
9020
9021 static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba,
9022                                          struct ufs_vreg *vreg)
9023 {
9024         if (!vreg)
9025                 return 0;
9026
9027         return ufshcd_config_vreg_load(hba->dev, vreg, vreg->max_uA);
9028 }
9029
9030 static int ufshcd_config_vreg(struct device *dev,
9031                 struct ufs_vreg *vreg, bool on)
9032 {
9033         if (regulator_count_voltages(vreg->reg) <= 0)
9034                 return 0;
9035
9036         return ufshcd_config_vreg_load(dev, vreg, on ? vreg->max_uA : 0);
9037 }
9038
9039 static int ufshcd_enable_vreg(struct device *dev, struct ufs_vreg *vreg)
9040 {
9041         int ret = 0;
9042
9043         if (!vreg || vreg->enabled)
9044                 goto out;
9045
9046         ret = ufshcd_config_vreg(dev, vreg, true);
9047         if (!ret)
9048                 ret = regulator_enable(vreg->reg);
9049
9050         if (!ret)
9051                 vreg->enabled = true;
9052         else
9053                 dev_err(dev, "%s: %s enable failed, err=%d\n",
9054                                 __func__, vreg->name, ret);
9055 out:
9056         return ret;
9057 }
9058
9059 static int ufshcd_disable_vreg(struct device *dev, struct ufs_vreg *vreg)
9060 {
9061         int ret = 0;
9062
9063         if (!vreg || !vreg->enabled || vreg->always_on)
9064                 goto out;
9065
9066         ret = regulator_disable(vreg->reg);
9067
9068         if (!ret) {
9069                 /* ignore errors on applying disable config */
9070                 ufshcd_config_vreg(dev, vreg, false);
9071                 vreg->enabled = false;
9072         } else {
9073                 dev_err(dev, "%s: %s disable failed, err=%d\n",
9074                                 __func__, vreg->name, ret);
9075         }
9076 out:
9077         return ret;
9078 }
9079
9080 static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on)
9081 {
9082         int ret = 0;
9083         struct device *dev = hba->dev;
9084         struct ufs_vreg_info *info = &hba->vreg_info;
9085
9086         ret = ufshcd_toggle_vreg(dev, info->vcc, on);
9087         if (ret)
9088                 goto out;
9089
9090         ret = ufshcd_toggle_vreg(dev, info->vccq, on);
9091         if (ret)
9092                 goto out;
9093
9094         ret = ufshcd_toggle_vreg(dev, info->vccq2, on);
9095
9096 out:
9097         if (ret) {
9098                 ufshcd_toggle_vreg(dev, info->vccq2, false);
9099                 ufshcd_toggle_vreg(dev, info->vccq, false);
9100                 ufshcd_toggle_vreg(dev, info->vcc, false);
9101         }
9102         return ret;
9103 }
9104
9105 static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on)
9106 {
9107         struct ufs_vreg_info *info = &hba->vreg_info;
9108
9109         return ufshcd_toggle_vreg(hba->dev, info->vdd_hba, on);
9110 }
9111
9112 int ufshcd_get_vreg(struct device *dev, struct ufs_vreg *vreg)
9113 {
9114         int ret = 0;
9115
9116         if (!vreg)
9117                 goto out;
9118
9119         vreg->reg = devm_regulator_get(dev, vreg->name);
9120         if (IS_ERR(vreg->reg)) {
9121                 ret = PTR_ERR(vreg->reg);
9122                 dev_err(dev, "%s: %s get failed, err=%d\n",
9123                                 __func__, vreg->name, ret);
9124         }
9125 out:
9126         return ret;
9127 }
9128 EXPORT_SYMBOL_GPL(ufshcd_get_vreg);
9129
9130 static int ufshcd_init_vreg(struct ufs_hba *hba)
9131 {
9132         int ret = 0;
9133         struct device *dev = hba->dev;
9134         struct ufs_vreg_info *info = &hba->vreg_info;
9135
9136         ret = ufshcd_get_vreg(dev, info->vcc);
9137         if (ret)
9138                 goto out;
9139
9140         ret = ufshcd_get_vreg(dev, info->vccq);
9141         if (!ret)
9142                 ret = ufshcd_get_vreg(dev, info->vccq2);
9143 out:
9144         return ret;
9145 }
9146
9147 static int ufshcd_init_hba_vreg(struct ufs_hba *hba)
9148 {
9149         struct ufs_vreg_info *info = &hba->vreg_info;
9150
9151         return ufshcd_get_vreg(hba->dev, info->vdd_hba);
9152 }
9153
9154 static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on)
9155 {
9156         int ret = 0;
9157         struct ufs_clk_info *clki;
9158         struct list_head *head = &hba->clk_list_head;
9159         unsigned long flags;
9160         ktime_t start = ktime_get();
9161         bool clk_state_changed = false;
9162
9163         if (list_empty(head))
9164                 goto out;
9165
9166         ret = ufshcd_vops_setup_clocks(hba, on, PRE_CHANGE);
9167         if (ret)
9168                 return ret;
9169
9170         list_for_each_entry(clki, head, list) {
9171                 if (!IS_ERR_OR_NULL(clki->clk)) {
9172                         /*
9173                          * Don't disable clocks which are needed
9174                          * to keep the link active.
9175                          */
9176                         if (ufshcd_is_link_active(hba) &&
9177                             clki->keep_link_active)
9178                                 continue;
9179
9180                         clk_state_changed = on ^ clki->enabled;
9181                         if (on && !clki->enabled) {
9182                                 ret = clk_prepare_enable(clki->clk);
9183                                 if (ret) {
9184                                         dev_err(hba->dev, "%s: %s prepare enable failed, %d\n",
9185                                                 __func__, clki->name, ret);
9186                                         goto out;
9187                                 }
9188                         } else if (!on && clki->enabled) {
9189                                 clk_disable_unprepare(clki->clk);
9190                         }
9191                         clki->enabled = on;
9192                         dev_dbg(hba->dev, "%s: clk: %s %sabled\n", __func__,
9193                                         clki->name, on ? "en" : "dis");
9194                 }
9195         }
9196
9197         ret = ufshcd_vops_setup_clocks(hba, on, POST_CHANGE);
9198         if (ret)
9199                 return ret;
9200
9201         if (!ufshcd_is_clkscaling_supported(hba))
9202                 ufshcd_pm_qos_update(hba, on);
9203 out:
9204         if (ret) {
9205                 list_for_each_entry(clki, head, list) {
9206                         if (!IS_ERR_OR_NULL(clki->clk) && clki->enabled)
9207                                 clk_disable_unprepare(clki->clk);
9208                 }
9209         } else if (!ret && on) {
9210                 spin_lock_irqsave(hba->host->host_lock, flags);
9211                 hba->clk_gating.state = CLKS_ON;
9212                 trace_ufshcd_clk_gating(dev_name(hba->dev),
9213                                         hba->clk_gating.state);
9214                 spin_unlock_irqrestore(hba->host->host_lock, flags);
9215         }
9216
9217         if (clk_state_changed)
9218                 trace_ufshcd_profile_clk_gating(dev_name(hba->dev),
9219                         (on ? "on" : "off"),
9220                         ktime_to_us(ktime_sub(ktime_get(), start)), ret);
9221         return ret;
9222 }
9223
9224 static enum ufs_ref_clk_freq ufshcd_parse_ref_clk_property(struct ufs_hba *hba)
9225 {
9226         u32 freq;
9227         int ret = device_property_read_u32(hba->dev, "ref-clk-freq", &freq);
9228
9229         if (ret) {
9230                 dev_dbg(hba->dev, "Cannot query 'ref-clk-freq' property = %d", ret);
9231                 return REF_CLK_FREQ_INVAL;
9232         }
9233
9234         return ufs_get_bref_clk_from_hz(freq);
9235 }
9236
9237 static int ufshcd_init_clocks(struct ufs_hba *hba)
9238 {
9239         int ret = 0;
9240         struct ufs_clk_info *clki;
9241         struct device *dev = hba->dev;
9242         struct list_head *head = &hba->clk_list_head;
9243
9244         if (list_empty(head))
9245                 goto out;
9246
9247         list_for_each_entry(clki, head, list) {
9248                 if (!clki->name)
9249                         continue;
9250
9251                 clki->clk = devm_clk_get(dev, clki->name);
9252                 if (IS_ERR(clki->clk)) {
9253                         ret = PTR_ERR(clki->clk);
9254                         dev_err(dev, "%s: %s clk get failed, %d\n",
9255                                         __func__, clki->name, ret);
9256                         goto out;
9257                 }
9258
9259                 /*
9260                  * Parse device ref clk freq as per device tree "ref_clk".
9261                  * Default dev_ref_clk_freq is set to REF_CLK_FREQ_INVAL
9262                  * in ufshcd_alloc_host().
9263                  */
9264                 if (!strcmp(clki->name, "ref_clk"))
9265                         ufshcd_parse_dev_ref_clk_freq(hba, clki->clk);
9266
9267                 if (clki->max_freq) {
9268                         ret = clk_set_rate(clki->clk, clki->max_freq);
9269                         if (ret) {
9270                                 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
9271                                         __func__, clki->name,
9272                                         clki->max_freq, ret);
9273                                 goto out;
9274                         }
9275                         clki->curr_freq = clki->max_freq;
9276                 }
9277                 dev_dbg(dev, "%s: clk: %s, rate: %lu\n", __func__,
9278                                 clki->name, clk_get_rate(clki->clk));
9279         }
9280
9281         /* Set Max. frequency for all clocks */
9282         if (hba->use_pm_opp) {
9283                 ret = ufshcd_opp_set_rate(hba, ULONG_MAX);
9284                 if (ret) {
9285                         dev_err(hba->dev, "%s: failed to set OPP: %d", __func__,
9286                                 ret);
9287                         goto out;
9288                 }
9289         }
9290
9291 out:
9292         return ret;
9293 }
9294
9295 static int ufshcd_variant_hba_init(struct ufs_hba *hba)
9296 {
9297         int err = 0;
9298
9299         if (!hba->vops)
9300                 goto out;
9301
9302         err = ufshcd_vops_init(hba);
9303         if (err)
9304                 dev_err_probe(hba->dev, err,
9305                               "%s: variant %s init failed with err %d\n",
9306                               __func__, ufshcd_get_var_name(hba), err);
9307 out:
9308         return err;
9309 }
9310
9311 static void ufshcd_variant_hba_exit(struct ufs_hba *hba)
9312 {
9313         if (!hba->vops)
9314                 return;
9315
9316         ufshcd_vops_exit(hba);
9317 }
9318
9319 static int ufshcd_hba_init(struct ufs_hba *hba)
9320 {
9321         int err;
9322
9323         /*
9324          * Handle host controller power separately from the UFS device power
9325          * rails as it will help controlling the UFS host controller power
9326          * collapse easily which is different than UFS device power collapse.
9327          * Also, enable the host controller power before we go ahead with rest
9328          * of the initialization here.
9329          */
9330         err = ufshcd_init_hba_vreg(hba);
9331         if (err)
9332                 goto out;
9333
9334         err = ufshcd_setup_hba_vreg(hba, true);
9335         if (err)
9336                 goto out;
9337
9338         err = ufshcd_init_clocks(hba);
9339         if (err)
9340                 goto out_disable_hba_vreg;
9341
9342         if (hba->dev_ref_clk_freq == REF_CLK_FREQ_INVAL)
9343                 hba->dev_ref_clk_freq = ufshcd_parse_ref_clk_property(hba);
9344
9345         err = ufshcd_setup_clocks(hba, true);
9346         if (err)
9347                 goto out_disable_hba_vreg;
9348
9349         err = ufshcd_init_vreg(hba);
9350         if (err)
9351                 goto out_disable_clks;
9352
9353         err = ufshcd_setup_vreg(hba, true);
9354         if (err)
9355                 goto out_disable_clks;
9356
9357         err = ufshcd_variant_hba_init(hba);
9358         if (err)
9359                 goto out_disable_vreg;
9360
9361         ufs_debugfs_hba_init(hba);
9362         ufs_fault_inject_hba_init(hba);
9363
9364         hba->is_powered = true;
9365         goto out;
9366
9367 out_disable_vreg:
9368         ufshcd_setup_vreg(hba, false);
9369 out_disable_clks:
9370         ufshcd_setup_clocks(hba, false);
9371 out_disable_hba_vreg:
9372         ufshcd_setup_hba_vreg(hba, false);
9373 out:
9374         return err;
9375 }
9376
9377 static void ufshcd_hba_exit(struct ufs_hba *hba)
9378 {
9379         if (hba->is_powered) {
9380                 ufshcd_pm_qos_exit(hba);
9381                 ufshcd_exit_clk_scaling(hba);
9382                 ufshcd_exit_clk_gating(hba);
9383                 if (hba->eh_wq)
9384                         destroy_workqueue(hba->eh_wq);
9385                 ufs_debugfs_hba_exit(hba);
9386                 ufshcd_variant_hba_exit(hba);
9387                 ufshcd_setup_vreg(hba, false);
9388                 ufshcd_setup_clocks(hba, false);
9389                 ufshcd_setup_hba_vreg(hba, false);
9390                 hba->is_powered = false;
9391                 ufs_put_device_desc(hba);
9392         }
9393 }
9394
9395 static int ufshcd_execute_start_stop(struct scsi_device *sdev,
9396                                      enum ufs_dev_pwr_mode pwr_mode,
9397                                      struct scsi_sense_hdr *sshdr)
9398 {
9399         const unsigned char cdb[6] = { START_STOP, 0, 0, 0, pwr_mode << 4, 0 };
9400         struct scsi_failure failure_defs[] = {
9401                 {
9402                         .allowed = 2,
9403                         .result = SCMD_FAILURE_RESULT_ANY,
9404                 },
9405         };
9406         struct scsi_failures failures = {
9407                 .failure_definitions = failure_defs,
9408         };
9409         const struct scsi_exec_args args = {
9410                 .failures = &failures,
9411                 .sshdr = sshdr,
9412                 .req_flags = BLK_MQ_REQ_PM,
9413                 .scmd_flags = SCMD_FAIL_IF_RECOVERING,
9414         };
9415
9416         return scsi_execute_cmd(sdev, cdb, REQ_OP_DRV_IN, /*buffer=*/NULL,
9417                         /*bufflen=*/0, /*timeout=*/10 * HZ, /*retries=*/0,
9418                         &args);
9419 }
9420
9421 /**
9422  * ufshcd_set_dev_pwr_mode - sends START STOP UNIT command to set device
9423  *                           power mode
9424  * @hba: per adapter instance
9425  * @pwr_mode: device power mode to set
9426  *
9427  * Return: 0 if requested power mode is set successfully;
9428  *         < 0 if failed to set the requested power mode.
9429  */
9430 static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba,
9431                                      enum ufs_dev_pwr_mode pwr_mode)
9432 {
9433         struct scsi_sense_hdr sshdr;
9434         struct scsi_device *sdp;
9435         unsigned long flags;
9436         int ret;
9437
9438         spin_lock_irqsave(hba->host->host_lock, flags);
9439         sdp = hba->ufs_device_wlun;
9440         if (sdp && scsi_device_online(sdp))
9441                 ret = scsi_device_get(sdp);
9442         else
9443                 ret = -ENODEV;
9444         spin_unlock_irqrestore(hba->host->host_lock, flags);
9445
9446         if (ret)
9447                 return ret;
9448
9449         /*
9450          * If scsi commands fail, the scsi mid-layer schedules scsi error-
9451          * handling, which would wait for host to be resumed. Since we know
9452          * we are functional while we are here, skip host resume in error
9453          * handling context.
9454          */
9455         hba->host->eh_noresume = 1;
9456
9457         /*
9458          * Current function would be generally called from the power management
9459          * callbacks hence set the RQF_PM flag so that it doesn't resume the
9460          * already suspended childs.
9461          */
9462         ret = ufshcd_execute_start_stop(sdp, pwr_mode, &sshdr);
9463         if (ret) {
9464                 sdev_printk(KERN_WARNING, sdp,
9465                             "START_STOP failed for power mode: %d, result %x\n",
9466                             pwr_mode, ret);
9467                 if (ret > 0) {
9468                         if (scsi_sense_valid(&sshdr))
9469                                 scsi_print_sense_hdr(sdp, NULL, &sshdr);
9470                         ret = -EIO;
9471                 }
9472         } else {
9473                 hba->curr_dev_pwr_mode = pwr_mode;
9474         }
9475
9476         scsi_device_put(sdp);
9477         hba->host->eh_noresume = 0;
9478         return ret;
9479 }
9480
9481 static int ufshcd_link_state_transition(struct ufs_hba *hba,
9482                                         enum uic_link_state req_link_state,
9483                                         bool check_for_bkops)
9484 {
9485         int ret = 0;
9486
9487         if (req_link_state == hba->uic_link_state)
9488                 return 0;
9489
9490         if (req_link_state == UIC_LINK_HIBERN8_STATE) {
9491                 ret = ufshcd_uic_hibern8_enter(hba);
9492                 if (!ret) {
9493                         ufshcd_set_link_hibern8(hba);
9494                 } else {
9495                         dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
9496                                         __func__, ret);
9497                         goto out;
9498                 }
9499         }
9500         /*
9501          * If autobkops is enabled, link can't be turned off because
9502          * turning off the link would also turn off the device, except in the
9503          * case of DeepSleep where the device is expected to remain powered.
9504          */
9505         else if ((req_link_state == UIC_LINK_OFF_STATE) &&
9506                  (!check_for_bkops || !hba->auto_bkops_enabled)) {
9507                 /*
9508                  * Let's make sure that link is in low power mode, we are doing
9509                  * this currently by putting the link in Hibern8. Otherway to
9510                  * put the link in low power mode is to send the DME end point
9511                  * to device and then send the DME reset command to local
9512                  * unipro. But putting the link in hibern8 is much faster.
9513                  *
9514                  * Note also that putting the link in Hibern8 is a requirement
9515                  * for entering DeepSleep.
9516                  */
9517                 ret = ufshcd_uic_hibern8_enter(hba);
9518                 if (ret) {
9519                         dev_err(hba->dev, "%s: hibern8 enter failed %d\n",
9520                                         __func__, ret);
9521                         goto out;
9522                 }
9523                 /*
9524                  * Change controller state to "reset state" which
9525                  * should also put the link in off/reset state
9526                  */
9527                 ufshcd_hba_stop(hba);
9528                 /*
9529                  * TODO: Check if we need any delay to make sure that
9530                  * controller is reset
9531                  */
9532                 ufshcd_set_link_off(hba);
9533         }
9534
9535 out:
9536         return ret;
9537 }
9538
9539 static void ufshcd_vreg_set_lpm(struct ufs_hba *hba)
9540 {
9541         bool vcc_off = false;
9542
9543         /*
9544          * It seems some UFS devices may keep drawing more than sleep current
9545          * (atleast for 500us) from UFS rails (especially from VCCQ rail).
9546          * To avoid this situation, add 2ms delay before putting these UFS
9547          * rails in LPM mode.
9548          */
9549         if (!ufshcd_is_link_active(hba) &&
9550             hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM)
9551                 usleep_range(2000, 2100);
9552
9553         /*
9554          * If UFS device is either in UFS_Sleep turn off VCC rail to save some
9555          * power.
9556          *
9557          * If UFS device and link is in OFF state, all power supplies (VCC,
9558          * VCCQ, VCCQ2) can be turned off if power on write protect is not
9559          * required. If UFS link is inactive (Hibern8 or OFF state) and device
9560          * is in sleep state, put VCCQ & VCCQ2 rails in LPM mode.
9561          *
9562          * Ignore the error returned by ufshcd_toggle_vreg() as device is anyway
9563          * in low power state which would save some power.
9564          *
9565          * If Write Booster is enabled and the device needs to flush the WB
9566          * buffer OR if bkops status is urgent for WB, keep Vcc on.
9567          */
9568         if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
9569             !hba->dev_info.is_lu_power_on_wp) {
9570                 ufshcd_setup_vreg(hba, false);
9571                 vcc_off = true;
9572         } else if (!ufshcd_is_ufs_dev_active(hba)) {
9573                 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
9574                 vcc_off = true;
9575                 if (ufshcd_is_link_hibern8(hba) || ufshcd_is_link_off(hba)) {
9576                         ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
9577                         ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq2);
9578                 }
9579         }
9580
9581         /*
9582          * Some UFS devices require delay after VCC power rail is turned-off.
9583          */
9584         if (vcc_off && hba->vreg_info.vcc &&
9585                 hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_AFTER_LPM)
9586                 usleep_range(5000, 5100);
9587 }
9588
9589 #ifdef CONFIG_PM
9590 static int ufshcd_vreg_set_hpm(struct ufs_hba *hba)
9591 {
9592         int ret = 0;
9593
9594         if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
9595             !hba->dev_info.is_lu_power_on_wp) {
9596                 ret = ufshcd_setup_vreg(hba, true);
9597         } else if (!ufshcd_is_ufs_dev_active(hba)) {
9598                 if (!ufshcd_is_link_active(hba)) {
9599                         ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq);
9600                         if (ret)
9601                                 goto vcc_disable;
9602                         ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2);
9603                         if (ret)
9604                                 goto vccq_lpm;
9605                 }
9606                 ret = ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, true);
9607         }
9608         goto out;
9609
9610 vccq_lpm:
9611         ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
9612 vcc_disable:
9613         ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
9614 out:
9615         return ret;
9616 }
9617 #endif /* CONFIG_PM */
9618
9619 static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba)
9620 {
9621         if (ufshcd_is_link_off(hba) || ufshcd_can_aggressive_pc(hba))
9622                 ufshcd_setup_hba_vreg(hba, false);
9623 }
9624
9625 static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba)
9626 {
9627         if (ufshcd_is_link_off(hba) || ufshcd_can_aggressive_pc(hba))
9628                 ufshcd_setup_hba_vreg(hba, true);
9629 }
9630
9631 static int __ufshcd_wl_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op)
9632 {
9633         int ret = 0;
9634         bool check_for_bkops;
9635         enum ufs_pm_level pm_lvl;
9636         enum ufs_dev_pwr_mode req_dev_pwr_mode;
9637         enum uic_link_state req_link_state;
9638
9639         hba->pm_op_in_progress = true;
9640         if (pm_op != UFS_SHUTDOWN_PM) {
9641                 pm_lvl = pm_op == UFS_RUNTIME_PM ?
9642                          hba->rpm_lvl : hba->spm_lvl;
9643                 req_dev_pwr_mode = ufs_get_pm_lvl_to_dev_pwr_mode(pm_lvl);
9644                 req_link_state = ufs_get_pm_lvl_to_link_pwr_state(pm_lvl);
9645         } else {
9646                 req_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE;
9647                 req_link_state = UIC_LINK_OFF_STATE;
9648         }
9649
9650         /*
9651          * If we can't transition into any of the low power modes
9652          * just gate the clocks.
9653          */
9654         ufshcd_hold(hba);
9655         hba->clk_gating.is_suspended = true;
9656
9657         if (ufshcd_is_clkscaling_supported(hba))
9658                 ufshcd_clk_scaling_suspend(hba, true);
9659
9660         if (req_dev_pwr_mode == UFS_ACTIVE_PWR_MODE &&
9661                         req_link_state == UIC_LINK_ACTIVE_STATE) {
9662                 goto vops_suspend;
9663         }
9664
9665         if ((req_dev_pwr_mode == hba->curr_dev_pwr_mode) &&
9666             (req_link_state == hba->uic_link_state))
9667                 goto enable_scaling;
9668
9669         /* UFS device & link must be active before we enter in this function */
9670         if (!ufshcd_is_ufs_dev_active(hba) || !ufshcd_is_link_active(hba)) {
9671                 /*  Wait err handler finish or trigger err recovery */
9672                 if (!ufshcd_eh_in_progress(hba))
9673                         ufshcd_force_error_recovery(hba);
9674                 ret = -EBUSY;
9675                 goto enable_scaling;
9676         }
9677
9678         if (pm_op == UFS_RUNTIME_PM) {
9679                 if (ufshcd_can_autobkops_during_suspend(hba)) {
9680                         /*
9681                          * The device is idle with no requests in the queue,
9682                          * allow background operations if bkops status shows
9683                          * that performance might be impacted.
9684                          */
9685                         ret = ufshcd_bkops_ctrl(hba);
9686                         if (ret) {
9687                                 /*
9688                                  * If return err in suspend flow, IO will hang.
9689                                  * Trigger error handler and break suspend for
9690                                  * error recovery.
9691                                  */
9692                                 ufshcd_force_error_recovery(hba);
9693                                 ret = -EBUSY;
9694                                 goto enable_scaling;
9695                         }
9696                 } else {
9697                         /* make sure that auto bkops is disabled */
9698                         ufshcd_disable_auto_bkops(hba);
9699                 }
9700                 /*
9701                  * If device needs to do BKOP or WB buffer flush during
9702                  * Hibern8, keep device power mode as "active power mode"
9703                  * and VCC supply.
9704                  */
9705                 hba->dev_info.b_rpm_dev_flush_capable =
9706                         hba->auto_bkops_enabled ||
9707                         (((req_link_state == UIC_LINK_HIBERN8_STATE) ||
9708                         ((req_link_state == UIC_LINK_ACTIVE_STATE) &&
9709                         ufshcd_is_auto_hibern8_enabled(hba))) &&
9710                         ufshcd_wb_need_flush(hba));
9711         }
9712
9713         flush_work(&hba->eeh_work);
9714
9715         ret = ufshcd_vops_suspend(hba, pm_op, PRE_CHANGE);
9716         if (ret)
9717                 goto enable_scaling;
9718
9719         if (req_dev_pwr_mode != hba->curr_dev_pwr_mode) {
9720                 if (pm_op != UFS_RUNTIME_PM)
9721                         /* ensure that bkops is disabled */
9722                         ufshcd_disable_auto_bkops(hba);
9723
9724                 if (!hba->dev_info.b_rpm_dev_flush_capable) {
9725                         ret = ufshcd_set_dev_pwr_mode(hba, req_dev_pwr_mode);
9726                         if (ret && pm_op != UFS_SHUTDOWN_PM) {
9727                                 /*
9728                                  * If return err in suspend flow, IO will hang.
9729                                  * Trigger error handler and break suspend for
9730                                  * error recovery.
9731                                  */
9732                                 ufshcd_force_error_recovery(hba);
9733                                 ret = -EBUSY;
9734                         }
9735                         if (ret)
9736                                 goto enable_scaling;
9737                 }
9738         }
9739
9740         /*
9741          * In the case of DeepSleep, the device is expected to remain powered
9742          * with the link off, so do not check for bkops.
9743          */
9744         check_for_bkops = !ufshcd_is_ufs_dev_deepsleep(hba);
9745         ret = ufshcd_link_state_transition(hba, req_link_state, check_for_bkops);
9746         if (ret && pm_op != UFS_SHUTDOWN_PM) {
9747                 /*
9748                  * If return err in suspend flow, IO will hang.
9749                  * Trigger error handler and break suspend for
9750                  * error recovery.
9751                  */
9752                 ufshcd_force_error_recovery(hba);
9753                 ret = -EBUSY;
9754         }
9755         if (ret)
9756                 goto set_dev_active;
9757
9758 vops_suspend:
9759         /*
9760          * Call vendor specific suspend callback. As these callbacks may access
9761          * vendor specific host controller register space call them before the
9762          * host clocks are ON.
9763          */
9764         ret = ufshcd_vops_suspend(hba, pm_op, POST_CHANGE);
9765         if (ret)
9766                 goto set_link_active;
9767
9768         cancel_delayed_work_sync(&hba->ufs_rtc_update_work);
9769         goto out;
9770
9771 set_link_active:
9772         /*
9773          * Device hardware reset is required to exit DeepSleep. Also, for
9774          * DeepSleep, the link is off so host reset and restore will be done
9775          * further below.
9776          */
9777         if (ufshcd_is_ufs_dev_deepsleep(hba)) {
9778                 ufshcd_device_reset(hba);
9779                 WARN_ON(!ufshcd_is_link_off(hba));
9780         }
9781         if (ufshcd_is_link_hibern8(hba) && !ufshcd_uic_hibern8_exit(hba))
9782                 ufshcd_set_link_active(hba);
9783         else if (ufshcd_is_link_off(hba))
9784                 ufshcd_host_reset_and_restore(hba);
9785 set_dev_active:
9786         /* Can also get here needing to exit DeepSleep */
9787         if (ufshcd_is_ufs_dev_deepsleep(hba)) {
9788                 ufshcd_device_reset(hba);
9789                 ufshcd_host_reset_and_restore(hba);
9790         }
9791         if (!ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE))
9792                 ufshcd_disable_auto_bkops(hba);
9793 enable_scaling:
9794         if (ufshcd_is_clkscaling_supported(hba))
9795                 ufshcd_clk_scaling_suspend(hba, false);
9796
9797         hba->dev_info.b_rpm_dev_flush_capable = false;
9798 out:
9799         if (hba->dev_info.b_rpm_dev_flush_capable) {
9800                 schedule_delayed_work(&hba->rpm_dev_flush_recheck_work,
9801                         msecs_to_jiffies(RPM_DEV_FLUSH_RECHECK_WORK_DELAY_MS));
9802         }
9803
9804         if (ret) {
9805                 ufshcd_update_evt_hist(hba, UFS_EVT_WL_SUSP_ERR, (u32)ret);
9806                 hba->clk_gating.is_suspended = false;
9807                 ufshcd_release(hba);
9808         }
9809         hba->pm_op_in_progress = false;
9810         return ret;
9811 }
9812
9813 #ifdef CONFIG_PM
9814 static int __ufshcd_wl_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
9815 {
9816         int ret;
9817         enum uic_link_state old_link_state = hba->uic_link_state;
9818
9819         hba->pm_op_in_progress = true;
9820
9821         /*
9822          * Call vendor specific resume callback. As these callbacks may access
9823          * vendor specific host controller register space call them when the
9824          * host clocks are ON.
9825          */
9826         ret = ufshcd_vops_resume(hba, pm_op);
9827         if (ret)
9828                 goto out;
9829
9830         /* For DeepSleep, the only supported option is to have the link off */
9831         WARN_ON(ufshcd_is_ufs_dev_deepsleep(hba) && !ufshcd_is_link_off(hba));
9832
9833         if (ufshcd_is_link_hibern8(hba)) {
9834                 ret = ufshcd_uic_hibern8_exit(hba);
9835                 if (!ret) {
9836                         ufshcd_set_link_active(hba);
9837                 } else {
9838                         dev_err(hba->dev, "%s: hibern8 exit failed %d\n",
9839                                         __func__, ret);
9840                         goto vendor_suspend;
9841                 }
9842         } else if (ufshcd_is_link_off(hba)) {
9843                 /*
9844                  * A full initialization of the host and the device is
9845                  * required since the link was put to off during suspend.
9846                  * Note, in the case of DeepSleep, the device will exit
9847                  * DeepSleep due to device reset.
9848                  */
9849                 ret = ufshcd_reset_and_restore(hba);
9850                 /*
9851                  * ufshcd_reset_and_restore() should have already
9852                  * set the link state as active
9853                  */
9854                 if (ret || !ufshcd_is_link_active(hba))
9855                         goto vendor_suspend;
9856         }
9857
9858         if (!ufshcd_is_ufs_dev_active(hba)) {
9859                 ret = ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE);
9860                 if (ret)
9861                         goto set_old_link_state;
9862                 ufshcd_set_timestamp_attr(hba);
9863                 schedule_delayed_work(&hba->ufs_rtc_update_work,
9864                                       msecs_to_jiffies(UFS_RTC_UPDATE_INTERVAL_MS));
9865         }
9866
9867         if (ufshcd_keep_autobkops_enabled_except_suspend(hba))
9868                 ufshcd_enable_auto_bkops(hba);
9869         else
9870                 /*
9871                  * If BKOPs operations are urgently needed at this moment then
9872                  * keep auto-bkops enabled or else disable it.
9873                  */
9874                 ufshcd_bkops_ctrl(hba);
9875
9876         if (hba->ee_usr_mask)
9877                 ufshcd_write_ee_control(hba);
9878
9879         if (ufshcd_is_clkscaling_supported(hba))
9880                 ufshcd_clk_scaling_suspend(hba, false);
9881
9882         if (hba->dev_info.b_rpm_dev_flush_capable) {
9883                 hba->dev_info.b_rpm_dev_flush_capable = false;
9884                 cancel_delayed_work(&hba->rpm_dev_flush_recheck_work);
9885         }
9886
9887         ufshcd_configure_auto_hibern8(hba);
9888
9889         goto out;
9890
9891 set_old_link_state:
9892         ufshcd_link_state_transition(hba, old_link_state, 0);
9893 vendor_suspend:
9894         ufshcd_vops_suspend(hba, pm_op, PRE_CHANGE);
9895         ufshcd_vops_suspend(hba, pm_op, POST_CHANGE);
9896 out:
9897         if (ret)
9898                 ufshcd_update_evt_hist(hba, UFS_EVT_WL_RES_ERR, (u32)ret);
9899         hba->clk_gating.is_suspended = false;
9900         ufshcd_release(hba);
9901         hba->pm_op_in_progress = false;
9902         return ret;
9903 }
9904
9905 static int ufshcd_wl_runtime_suspend(struct device *dev)
9906 {
9907         struct scsi_device *sdev = to_scsi_device(dev);
9908         struct ufs_hba *hba;
9909         int ret;
9910         ktime_t start = ktime_get();
9911
9912         hba = shost_priv(sdev->host);
9913
9914         ret = __ufshcd_wl_suspend(hba, UFS_RUNTIME_PM);
9915         if (ret)
9916                 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret);
9917
9918         trace_ufshcd_wl_runtime_suspend(dev_name(dev), ret,
9919                 ktime_to_us(ktime_sub(ktime_get(), start)),
9920                 hba->curr_dev_pwr_mode, hba->uic_link_state);
9921
9922         return ret;
9923 }
9924
9925 static int ufshcd_wl_runtime_resume(struct device *dev)
9926 {
9927         struct scsi_device *sdev = to_scsi_device(dev);
9928         struct ufs_hba *hba;
9929         int ret = 0;
9930         ktime_t start = ktime_get();
9931
9932         hba = shost_priv(sdev->host);
9933
9934         ret = __ufshcd_wl_resume(hba, UFS_RUNTIME_PM);
9935         if (ret)
9936                 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret);
9937
9938         trace_ufshcd_wl_runtime_resume(dev_name(dev), ret,
9939                 ktime_to_us(ktime_sub(ktime_get(), start)),
9940                 hba->curr_dev_pwr_mode, hba->uic_link_state);
9941
9942         return ret;
9943 }
9944 #endif
9945
9946 #ifdef CONFIG_PM_SLEEP
9947 static int ufshcd_wl_suspend(struct device *dev)
9948 {
9949         struct scsi_device *sdev = to_scsi_device(dev);
9950         struct ufs_hba *hba;
9951         int ret = 0;
9952         ktime_t start = ktime_get();
9953
9954         hba = shost_priv(sdev->host);
9955         down(&hba->host_sem);
9956         hba->system_suspending = true;
9957
9958         if (pm_runtime_suspended(dev))
9959                 goto out;
9960
9961         ret = __ufshcd_wl_suspend(hba, UFS_SYSTEM_PM);
9962         if (ret) {
9963                 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__,  ret);
9964                 up(&hba->host_sem);
9965         }
9966
9967 out:
9968         if (!ret)
9969                 hba->is_sys_suspended = true;
9970         trace_ufshcd_wl_suspend(dev_name(dev), ret,
9971                 ktime_to_us(ktime_sub(ktime_get(), start)),
9972                 hba->curr_dev_pwr_mode, hba->uic_link_state);
9973
9974         return ret;
9975 }
9976
9977 static int ufshcd_wl_resume(struct device *dev)
9978 {
9979         struct scsi_device *sdev = to_scsi_device(dev);
9980         struct ufs_hba *hba;
9981         int ret = 0;
9982         ktime_t start = ktime_get();
9983
9984         hba = shost_priv(sdev->host);
9985
9986         if (pm_runtime_suspended(dev))
9987                 goto out;
9988
9989         ret = __ufshcd_wl_resume(hba, UFS_SYSTEM_PM);
9990         if (ret)
9991                 dev_err(&sdev->sdev_gendev, "%s failed: %d\n", __func__, ret);
9992 out:
9993         trace_ufshcd_wl_resume(dev_name(dev), ret,
9994                 ktime_to_us(ktime_sub(ktime_get(), start)),
9995                 hba->curr_dev_pwr_mode, hba->uic_link_state);
9996         if (!ret)
9997                 hba->is_sys_suspended = false;
9998         hba->system_suspending = false;
9999         up(&hba->host_sem);
10000         return ret;
10001 }
10002 #endif
10003
10004 /**
10005  * ufshcd_suspend - helper function for suspend operations
10006  * @hba: per adapter instance
10007  *
10008  * This function will put disable irqs, turn off clocks
10009  * and set vreg and hba-vreg in lpm mode.
10010  *
10011  * Return: 0 upon success; < 0 upon failure.
10012  */
10013 static int ufshcd_suspend(struct ufs_hba *hba)
10014 {
10015         int ret;
10016
10017         if (!hba->is_powered)
10018                 return 0;
10019         /*
10020          * Disable the host irq as host controller as there won't be any
10021          * host controller transaction expected till resume.
10022          */
10023         ufshcd_disable_irq(hba);
10024         ret = ufshcd_setup_clocks(hba, false);
10025         if (ret) {
10026                 ufshcd_enable_irq(hba);
10027                 return ret;
10028         }
10029         if (ufshcd_is_clkgating_allowed(hba)) {
10030                 hba->clk_gating.state = CLKS_OFF;
10031                 trace_ufshcd_clk_gating(dev_name(hba->dev),
10032                                         hba->clk_gating.state);
10033         }
10034
10035         ufshcd_vreg_set_lpm(hba);
10036         /* Put the host controller in low power mode if possible */
10037         ufshcd_hba_vreg_set_lpm(hba);
10038         ufshcd_pm_qos_update(hba, false);
10039         return ret;
10040 }
10041
10042 #ifdef CONFIG_PM
10043 /**
10044  * ufshcd_resume - helper function for resume operations
10045  * @hba: per adapter instance
10046  *
10047  * This function basically turns on the regulators, clocks and
10048  * irqs of the hba.
10049  *
10050  * Return: 0 for success and non-zero for failure.
10051  */
10052 static int ufshcd_resume(struct ufs_hba *hba)
10053 {
10054         int ret;
10055
10056         if (!hba->is_powered)
10057                 return 0;
10058
10059         ufshcd_hba_vreg_set_hpm(hba);
10060         ret = ufshcd_vreg_set_hpm(hba);
10061         if (ret)
10062                 goto out;
10063
10064         /* Make sure clocks are enabled before accessing controller */
10065         ret = ufshcd_setup_clocks(hba, true);
10066         if (ret)
10067                 goto disable_vreg;
10068
10069         /* enable the host irq as host controller would be active soon */
10070         ufshcd_enable_irq(hba);
10071
10072         goto out;
10073
10074 disable_vreg:
10075         ufshcd_vreg_set_lpm(hba);
10076 out:
10077         if (ret)
10078                 ufshcd_update_evt_hist(hba, UFS_EVT_RESUME_ERR, (u32)ret);
10079         return ret;
10080 }
10081 #endif /* CONFIG_PM */
10082
10083 #ifdef CONFIG_PM_SLEEP
10084 /**
10085  * ufshcd_system_suspend - system suspend callback
10086  * @dev: Device associated with the UFS controller.
10087  *
10088  * Executed before putting the system into a sleep state in which the contents
10089  * of main memory are preserved.
10090  *
10091  * Return: 0 for success and non-zero for failure.
10092  */
10093 int ufshcd_system_suspend(struct device *dev)
10094 {
10095         struct ufs_hba *hba = dev_get_drvdata(dev);
10096         int ret = 0;
10097         ktime_t start = ktime_get();
10098
10099         if (pm_runtime_suspended(hba->dev))
10100                 goto out;
10101
10102         ret = ufshcd_suspend(hba);
10103 out:
10104         trace_ufshcd_system_suspend(dev_name(hba->dev), ret,
10105                 ktime_to_us(ktime_sub(ktime_get(), start)),
10106                 hba->curr_dev_pwr_mode, hba->uic_link_state);
10107         return ret;
10108 }
10109 EXPORT_SYMBOL(ufshcd_system_suspend);
10110
10111 /**
10112  * ufshcd_system_resume - system resume callback
10113  * @dev: Device associated with the UFS controller.
10114  *
10115  * Executed after waking the system up from a sleep state in which the contents
10116  * of main memory were preserved.
10117  *
10118  * Return: 0 for success and non-zero for failure.
10119  */
10120 int ufshcd_system_resume(struct device *dev)
10121 {
10122         struct ufs_hba *hba = dev_get_drvdata(dev);
10123         ktime_t start = ktime_get();
10124         int ret = 0;
10125
10126         if (pm_runtime_suspended(hba->dev))
10127                 goto out;
10128
10129         ret = ufshcd_resume(hba);
10130
10131 out:
10132         trace_ufshcd_system_resume(dev_name(hba->dev), ret,
10133                 ktime_to_us(ktime_sub(ktime_get(), start)),
10134                 hba->curr_dev_pwr_mode, hba->uic_link_state);
10135
10136         return ret;
10137 }
10138 EXPORT_SYMBOL(ufshcd_system_resume);
10139 #endif /* CONFIG_PM_SLEEP */
10140
10141 #ifdef CONFIG_PM
10142 /**
10143  * ufshcd_runtime_suspend - runtime suspend callback
10144  * @dev: Device associated with the UFS controller.
10145  *
10146  * Check the description of ufshcd_suspend() function for more details.
10147  *
10148  * Return: 0 for success and non-zero for failure.
10149  */
10150 int ufshcd_runtime_suspend(struct device *dev)
10151 {
10152         struct ufs_hba *hba = dev_get_drvdata(dev);
10153         int ret;
10154         ktime_t start = ktime_get();
10155
10156         ret = ufshcd_suspend(hba);
10157
10158         trace_ufshcd_runtime_suspend(dev_name(hba->dev), ret,
10159                 ktime_to_us(ktime_sub(ktime_get(), start)),
10160                 hba->curr_dev_pwr_mode, hba->uic_link_state);
10161         return ret;
10162 }
10163 EXPORT_SYMBOL(ufshcd_runtime_suspend);
10164
10165 /**
10166  * ufshcd_runtime_resume - runtime resume routine
10167  * @dev: Device associated with the UFS controller.
10168  *
10169  * This function basically brings controller
10170  * to active state. Following operations are done in this function:
10171  *
10172  * 1. Turn on all the controller related clocks
10173  * 2. Turn ON VCC rail
10174  *
10175  * Return: 0 upon success; < 0 upon failure.
10176  */
10177 int ufshcd_runtime_resume(struct device *dev)
10178 {
10179         struct ufs_hba *hba = dev_get_drvdata(dev);
10180         int ret;
10181         ktime_t start = ktime_get();
10182
10183         ret = ufshcd_resume(hba);
10184
10185         trace_ufshcd_runtime_resume(dev_name(hba->dev), ret,
10186                 ktime_to_us(ktime_sub(ktime_get(), start)),
10187                 hba->curr_dev_pwr_mode, hba->uic_link_state);
10188         return ret;
10189 }
10190 EXPORT_SYMBOL(ufshcd_runtime_resume);
10191 #endif /* CONFIG_PM */
10192
10193 static void ufshcd_wl_shutdown(struct device *dev)
10194 {
10195         struct scsi_device *sdev = to_scsi_device(dev);
10196         struct ufs_hba *hba = shost_priv(sdev->host);
10197
10198         down(&hba->host_sem);
10199         hba->shutting_down = true;
10200         up(&hba->host_sem);
10201
10202         /* Turn on everything while shutting down */
10203         ufshcd_rpm_get_sync(hba);
10204         scsi_device_quiesce(sdev);
10205         shost_for_each_device(sdev, hba->host) {
10206                 if (sdev == hba->ufs_device_wlun)
10207                         continue;
10208                 mutex_lock(&sdev->state_mutex);
10209                 scsi_device_set_state(sdev, SDEV_OFFLINE);
10210                 mutex_unlock(&sdev->state_mutex);
10211         }
10212         __ufshcd_wl_suspend(hba, UFS_SHUTDOWN_PM);
10213
10214         /*
10215          * Next, turn off the UFS controller and the UFS regulators. Disable
10216          * clocks.
10217          */
10218         if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba))
10219                 ufshcd_suspend(hba);
10220
10221         hba->is_powered = false;
10222 }
10223
10224 /**
10225  * ufshcd_remove - de-allocate SCSI host and host memory space
10226  *              data structure memory
10227  * @hba: per adapter instance
10228  */
10229 void ufshcd_remove(struct ufs_hba *hba)
10230 {
10231         if (hba->ufs_device_wlun)
10232                 ufshcd_rpm_get_sync(hba);
10233         ufs_hwmon_remove(hba);
10234         ufs_bsg_remove(hba);
10235         ufs_sysfs_remove_nodes(hba->dev);
10236         cancel_delayed_work_sync(&hba->ufs_rtc_update_work);
10237         blk_mq_destroy_queue(hba->tmf_queue);
10238         blk_put_queue(hba->tmf_queue);
10239         blk_mq_free_tag_set(&hba->tmf_tag_set);
10240         if (hba->scsi_host_added)
10241                 scsi_remove_host(hba->host);
10242         /* disable interrupts */
10243         ufshcd_disable_intr(hba, hba->intr_mask);
10244         ufshcd_hba_stop(hba);
10245         ufshcd_hba_exit(hba);
10246 }
10247 EXPORT_SYMBOL_GPL(ufshcd_remove);
10248
10249 #ifdef CONFIG_PM_SLEEP
10250 int ufshcd_system_freeze(struct device *dev)
10251 {
10252
10253         return ufshcd_system_suspend(dev);
10254
10255 }
10256 EXPORT_SYMBOL_GPL(ufshcd_system_freeze);
10257
10258 int ufshcd_system_restore(struct device *dev)
10259 {
10260
10261         struct ufs_hba *hba = dev_get_drvdata(dev);
10262         int ret;
10263
10264         ret = ufshcd_system_resume(dev);
10265         if (ret)
10266                 return ret;
10267
10268         /* Configure UTRL and UTMRL base address registers */
10269         ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr),
10270                         REG_UTP_TRANSFER_REQ_LIST_BASE_L);
10271         ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr),
10272                         REG_UTP_TRANSFER_REQ_LIST_BASE_H);
10273         ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr),
10274                         REG_UTP_TASK_REQ_LIST_BASE_L);
10275         ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr),
10276                         REG_UTP_TASK_REQ_LIST_BASE_H);
10277         /*
10278          * Make sure that UTRL and UTMRL base address registers
10279          * are updated with the latest queue addresses. Only after
10280          * updating these addresses, we can queue the new commands.
10281          */
10282         ufshcd_readl(hba, REG_UTP_TASK_REQ_LIST_BASE_H);
10283
10284         return 0;
10285
10286 }
10287 EXPORT_SYMBOL_GPL(ufshcd_system_restore);
10288
10289 int ufshcd_system_thaw(struct device *dev)
10290 {
10291         return ufshcd_system_resume(dev);
10292 }
10293 EXPORT_SYMBOL_GPL(ufshcd_system_thaw);
10294 #endif /* CONFIG_PM_SLEEP  */
10295
10296 /**
10297  * ufshcd_dealloc_host - deallocate Host Bus Adapter (HBA)
10298  * @hba: pointer to Host Bus Adapter (HBA)
10299  */
10300 void ufshcd_dealloc_host(struct ufs_hba *hba)
10301 {
10302         scsi_host_put(hba->host);
10303 }
10304 EXPORT_SYMBOL_GPL(ufshcd_dealloc_host);
10305
10306 /**
10307  * ufshcd_set_dma_mask - Set dma mask based on the controller
10308  *                       addressing capability
10309  * @hba: per adapter instance
10310  *
10311  * Return: 0 for success, non-zero for failure.
10312  */
10313 static int ufshcd_set_dma_mask(struct ufs_hba *hba)
10314 {
10315         if (hba->vops && hba->vops->set_dma_mask)
10316                 return hba->vops->set_dma_mask(hba);
10317         if (hba->capabilities & MASK_64_ADDRESSING_SUPPORT) {
10318                 if (!dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(64)))
10319                         return 0;
10320         }
10321         return dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(32));
10322 }
10323
10324 /**
10325  * ufshcd_alloc_host - allocate Host Bus Adapter (HBA)
10326  * @dev: pointer to device handle
10327  * @hba_handle: driver private handle
10328  *
10329  * Return: 0 on success, non-zero value on failure.
10330  */
10331 int ufshcd_alloc_host(struct device *dev, struct ufs_hba **hba_handle)
10332 {
10333         struct Scsi_Host *host;
10334         struct ufs_hba *hba;
10335         int err = 0;
10336
10337         if (!dev) {
10338                 dev_err(dev,
10339                 "Invalid memory reference for dev is NULL\n");
10340                 err = -ENODEV;
10341                 goto out_error;
10342         }
10343
10344         host = scsi_host_alloc(&ufshcd_driver_template,
10345                                 sizeof(struct ufs_hba));
10346         if (!host) {
10347                 dev_err(dev, "scsi_host_alloc failed\n");
10348                 err = -ENOMEM;
10349                 goto out_error;
10350         }
10351         host->nr_maps = HCTX_TYPE_POLL + 1;
10352         hba = shost_priv(host);
10353         hba->host = host;
10354         hba->dev = dev;
10355         hba->dev_ref_clk_freq = REF_CLK_FREQ_INVAL;
10356         hba->nop_out_timeout = NOP_OUT_TIMEOUT;
10357         ufshcd_set_sg_entry_size(hba, sizeof(struct ufshcd_sg_entry));
10358         INIT_LIST_HEAD(&hba->clk_list_head);
10359         spin_lock_init(&hba->outstanding_lock);
10360
10361         *hba_handle = hba;
10362
10363 out_error:
10364         return err;
10365 }
10366 EXPORT_SYMBOL(ufshcd_alloc_host);
10367
10368 /* This function exists because blk_mq_alloc_tag_set() requires this. */
10369 static blk_status_t ufshcd_queue_tmf(struct blk_mq_hw_ctx *hctx,
10370                                      const struct blk_mq_queue_data *qd)
10371 {
10372         WARN_ON_ONCE(true);
10373         return BLK_STS_NOTSUPP;
10374 }
10375
10376 static const struct blk_mq_ops ufshcd_tmf_ops = {
10377         .queue_rq = ufshcd_queue_tmf,
10378 };
10379
10380 static int ufshcd_add_scsi_host(struct ufs_hba *hba)
10381 {
10382         int err;
10383
10384         if (is_mcq_supported(hba)) {
10385                 ufshcd_mcq_enable(hba);
10386                 err = ufshcd_alloc_mcq(hba);
10387                 if (!err) {
10388                         ufshcd_config_mcq(hba);
10389                 } else {
10390                         /* Continue with SDB mode */
10391                         ufshcd_mcq_disable(hba);
10392                         use_mcq_mode = false;
10393                         dev_err(hba->dev, "MCQ mode is disabled, err=%d\n",
10394                                 err);
10395                 }
10396         }
10397         if (!is_mcq_supported(hba) && !hba->lsdb_sup) {
10398                 dev_err(hba->dev,
10399                         "%s: failed to initialize (legacy doorbell mode not supported)\n",
10400                         __func__);
10401                 return -EINVAL;
10402         }
10403
10404         err = scsi_add_host(hba->host, hba->dev);
10405         if (err) {
10406                 dev_err(hba->dev, "scsi_add_host failed\n");
10407                 return err;
10408         }
10409         hba->scsi_host_added = true;
10410
10411         hba->tmf_tag_set = (struct blk_mq_tag_set) {
10412                 .nr_hw_queues   = 1,
10413                 .queue_depth    = hba->nutmrs,
10414                 .ops            = &ufshcd_tmf_ops,
10415                 .flags          = BLK_MQ_F_NO_SCHED,
10416         };
10417         err = blk_mq_alloc_tag_set(&hba->tmf_tag_set);
10418         if (err < 0)
10419                 goto remove_scsi_host;
10420         hba->tmf_queue = blk_mq_alloc_queue(&hba->tmf_tag_set, NULL, NULL);
10421         if (IS_ERR(hba->tmf_queue)) {
10422                 err = PTR_ERR(hba->tmf_queue);
10423                 goto free_tmf_tag_set;
10424         }
10425         hba->tmf_rqs = devm_kcalloc(hba->dev, hba->nutmrs,
10426                                     sizeof(*hba->tmf_rqs), GFP_KERNEL);
10427         if (!hba->tmf_rqs) {
10428                 err = -ENOMEM;
10429                 goto free_tmf_queue;
10430         }
10431
10432         return 0;
10433
10434 free_tmf_queue:
10435         blk_mq_destroy_queue(hba->tmf_queue);
10436         blk_put_queue(hba->tmf_queue);
10437
10438 free_tmf_tag_set:
10439         blk_mq_free_tag_set(&hba->tmf_tag_set);
10440
10441 remove_scsi_host:
10442         if (hba->scsi_host_added)
10443                 scsi_remove_host(hba->host);
10444
10445         return err;
10446 }
10447
10448 /**
10449  * ufshcd_init - Driver initialization routine
10450  * @hba: per-adapter instance
10451  * @mmio_base: base register address
10452  * @irq: Interrupt line of device
10453  *
10454  * Return: 0 on success, non-zero value on failure.
10455  */
10456 int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq)
10457 {
10458         int err;
10459         struct Scsi_Host *host = hba->host;
10460         struct device *dev = hba->dev;
10461
10462         /*
10463          * dev_set_drvdata() must be called before any callbacks are registered
10464          * that use dev_get_drvdata() (frequency scaling, clock scaling, hwmon,
10465          * sysfs).
10466          */
10467         dev_set_drvdata(dev, hba);
10468
10469         if (!mmio_base) {
10470                 dev_err(hba->dev,
10471                 "Invalid memory reference for mmio_base is NULL\n");
10472                 err = -ENODEV;
10473                 goto out_error;
10474         }
10475
10476         hba->mmio_base = mmio_base;
10477         hba->irq = irq;
10478         hba->vps = &ufs_hba_vps;
10479
10480         err = ufshcd_hba_init(hba);
10481         if (err)
10482                 goto out_error;
10483
10484         /* Read capabilities registers */
10485         err = ufshcd_hba_capabilities(hba);
10486         if (err)
10487                 goto out_disable;
10488
10489         /* Get UFS version supported by the controller */
10490         hba->ufs_version = ufshcd_get_ufs_version(hba);
10491
10492         /* Get Interrupt bit mask per version */
10493         hba->intr_mask = ufshcd_get_intr_mask(hba);
10494
10495         err = ufshcd_set_dma_mask(hba);
10496         if (err) {
10497                 dev_err(hba->dev, "set dma mask failed\n");
10498                 goto out_disable;
10499         }
10500
10501         /* Allocate memory for host memory space */
10502         err = ufshcd_memory_alloc(hba);
10503         if (err) {
10504                 dev_err(hba->dev, "Memory allocation failed\n");
10505                 goto out_disable;
10506         }
10507
10508         /* Configure LRB */
10509         ufshcd_host_memory_configure(hba);
10510
10511         host->can_queue = hba->nutrs - UFSHCD_NUM_RESERVED;
10512         host->cmd_per_lun = hba->nutrs - UFSHCD_NUM_RESERVED;
10513         host->max_id = UFSHCD_MAX_ID;
10514         host->max_lun = UFS_MAX_LUNS;
10515         host->max_channel = UFSHCD_MAX_CHANNEL;
10516         host->unique_id = host->host_no;
10517         host->max_cmd_len = UFS_CDB_SIZE;
10518         host->queuecommand_may_block = !!(hba->caps & UFSHCD_CAP_CLK_GATING);
10519
10520         /* Use default RPM delay if host not set */
10521         if (host->rpm_autosuspend_delay == 0)
10522                 host->rpm_autosuspend_delay = RPM_AUTOSUSPEND_DELAY_MS;
10523
10524         hba->max_pwr_info.is_valid = false;
10525
10526         /* Initialize work queues */
10527         hba->eh_wq = alloc_ordered_workqueue("ufs_eh_wq_%d", WQ_MEM_RECLAIM,
10528                                              hba->host->host_no);
10529         if (!hba->eh_wq) {
10530                 dev_err(hba->dev, "%s: failed to create eh workqueue\n",
10531                         __func__);
10532                 err = -ENOMEM;
10533                 goto out_disable;
10534         }
10535         INIT_WORK(&hba->eh_work, ufshcd_err_handler);
10536         INIT_WORK(&hba->eeh_work, ufshcd_exception_event_handler);
10537
10538         sema_init(&hba->host_sem, 1);
10539
10540         /* Initialize UIC command mutex */
10541         mutex_init(&hba->uic_cmd_mutex);
10542
10543         /* Initialize mutex for device management commands */
10544         mutex_init(&hba->dev_cmd.lock);
10545
10546         /* Initialize mutex for exception event control */
10547         mutex_init(&hba->ee_ctrl_mutex);
10548
10549         mutex_init(&hba->wb_mutex);
10550         init_rwsem(&hba->clk_scaling_lock);
10551
10552         ufshcd_init_clk_gating(hba);
10553
10554         ufshcd_init_clk_scaling(hba);
10555
10556         /*
10557          * In order to avoid any spurious interrupt immediately after
10558          * registering UFS controller interrupt handler, clear any pending UFS
10559          * interrupt status and disable all the UFS interrupts.
10560          */
10561         ufshcd_writel(hba, ufshcd_readl(hba, REG_INTERRUPT_STATUS),
10562                       REG_INTERRUPT_STATUS);
10563         ufshcd_writel(hba, 0, REG_INTERRUPT_ENABLE);
10564         /*
10565          * Make sure that UFS interrupts are disabled and any pending interrupt
10566          * status is cleared before registering UFS interrupt handler.
10567          */
10568         ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
10569
10570         /* IRQ registration */
10571         err = devm_request_irq(dev, irq, ufshcd_intr, IRQF_SHARED, UFSHCD, hba);
10572         if (err) {
10573                 dev_err(hba->dev, "request irq failed\n");
10574                 goto out_disable;
10575         } else {
10576                 hba->is_irq_enabled = true;
10577         }
10578
10579         /* Reset the attached device */
10580         ufshcd_device_reset(hba);
10581
10582         ufshcd_init_crypto(hba);
10583
10584         /* Host controller enable */
10585         err = ufshcd_hba_enable(hba);
10586         if (err) {
10587                 dev_err(hba->dev, "Host controller enable failed\n");
10588                 ufshcd_print_evt_hist(hba);
10589                 ufshcd_print_host_state(hba);
10590                 goto out_disable;
10591         }
10592
10593         /*
10594          * Set the default power management level for runtime and system PM.
10595          * Default power saving mode is to keep UFS link in Hibern8 state
10596          * and UFS device in sleep state.
10597          */
10598         hba->rpm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state(
10599                                                 UFS_SLEEP_PWR_MODE,
10600                                                 UIC_LINK_HIBERN8_STATE);
10601         hba->spm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state(
10602                                                 UFS_SLEEP_PWR_MODE,
10603                                                 UIC_LINK_HIBERN8_STATE);
10604
10605         INIT_DELAYED_WORK(&hba->rpm_dev_flush_recheck_work, ufshcd_rpm_dev_flush_recheck_work);
10606         INIT_DELAYED_WORK(&hba->ufs_rtc_update_work, ufshcd_rtc_work);
10607
10608         /* Set the default auto-hiberate idle timer value to 150 ms */
10609         if (ufshcd_is_auto_hibern8_supported(hba) && !hba->ahit) {
10610                 hba->ahit = FIELD_PREP(UFSHCI_AHIBERN8_TIMER_MASK, 150) |
10611                             FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, 3);
10612         }
10613
10614         /* Hold auto suspend until async scan completes */
10615         pm_runtime_get_sync(dev);
10616
10617         /*
10618          * We are assuming that device wasn't put in sleep/power-down
10619          * state exclusively during the boot stage before kernel.
10620          * This assumption helps avoid doing link startup twice during
10621          * ufshcd_probe_hba().
10622          */
10623         ufshcd_set_ufs_dev_active(hba);
10624
10625         /* Initialize hba, detect and initialize UFS device */
10626         ktime_t probe_start = ktime_get();
10627
10628         hba->ufshcd_state = UFSHCD_STATE_RESET;
10629
10630         err = ufshcd_link_startup(hba);
10631         if (err)
10632                 goto out_disable;
10633
10634         if (hba->quirks & UFSHCD_QUIRK_SKIP_PH_CONFIGURATION)
10635                 goto initialized;
10636
10637         /* Debug counters initialization */
10638         ufshcd_clear_dbg_ufs_stats(hba);
10639
10640         /* UniPro link is active now */
10641         ufshcd_set_link_active(hba);
10642
10643         /* Verify device initialization by sending NOP OUT UPIU */
10644         err = ufshcd_verify_dev_init(hba);
10645         if (err)
10646                 goto out_disable;
10647
10648         /* Initiate UFS initialization, and waiting until completion */
10649         err = ufshcd_complete_dev_init(hba);
10650         if (err)
10651                 goto out_disable;
10652
10653         err = ufshcd_device_params_init(hba);
10654         if (err)
10655                 goto out_disable;
10656
10657         err = ufshcd_post_device_init(hba);
10658
10659 initialized:
10660         ufshcd_process_probe_result(hba, probe_start, err);
10661         if (err)
10662                 goto out_disable;
10663
10664         err = ufshcd_add_scsi_host(hba);
10665         if (err)
10666                 goto out_disable;
10667
10668         async_schedule(ufshcd_async_scan, hba);
10669         ufs_sysfs_add_nodes(hba->dev);
10670
10671         device_enable_async_suspend(dev);
10672         ufshcd_pm_qos_init(hba);
10673         return 0;
10674
10675 out_disable:
10676         hba->is_irq_enabled = false;
10677         ufshcd_hba_exit(hba);
10678 out_error:
10679         return err;
10680 }
10681 EXPORT_SYMBOL_GPL(ufshcd_init);
10682
10683 void ufshcd_resume_complete(struct device *dev)
10684 {
10685         struct ufs_hba *hba = dev_get_drvdata(dev);
10686
10687         if (hba->complete_put) {
10688                 ufshcd_rpm_put(hba);
10689                 hba->complete_put = false;
10690         }
10691 }
10692 EXPORT_SYMBOL_GPL(ufshcd_resume_complete);
10693
10694 static bool ufshcd_rpm_ok_for_spm(struct ufs_hba *hba)
10695 {
10696         struct device *dev = &hba->ufs_device_wlun->sdev_gendev;
10697         enum ufs_dev_pwr_mode dev_pwr_mode;
10698         enum uic_link_state link_state;
10699         unsigned long flags;
10700         bool res;
10701
10702         spin_lock_irqsave(&dev->power.lock, flags);
10703         dev_pwr_mode = ufs_get_pm_lvl_to_dev_pwr_mode(hba->spm_lvl);
10704         link_state = ufs_get_pm_lvl_to_link_pwr_state(hba->spm_lvl);
10705         res = pm_runtime_suspended(dev) &&
10706               hba->curr_dev_pwr_mode == dev_pwr_mode &&
10707               hba->uic_link_state == link_state &&
10708               !hba->dev_info.b_rpm_dev_flush_capable;
10709         spin_unlock_irqrestore(&dev->power.lock, flags);
10710
10711         return res;
10712 }
10713
10714 int __ufshcd_suspend_prepare(struct device *dev, bool rpm_ok_for_spm)
10715 {
10716         struct ufs_hba *hba = dev_get_drvdata(dev);
10717         int ret;
10718
10719         /*
10720          * SCSI assumes that runtime-pm and system-pm for scsi drivers
10721          * are same. And it doesn't wake up the device for system-suspend
10722          * if it's runtime suspended. But ufs doesn't follow that.
10723          * Refer ufshcd_resume_complete()
10724          */
10725         if (hba->ufs_device_wlun) {
10726                 /* Prevent runtime suspend */
10727                 ufshcd_rpm_get_noresume(hba);
10728                 /*
10729                  * Check if already runtime suspended in same state as system
10730                  * suspend would be.
10731                  */
10732                 if (!rpm_ok_for_spm || !ufshcd_rpm_ok_for_spm(hba)) {
10733                         /* RPM state is not ok for SPM, so runtime resume */
10734                         ret = ufshcd_rpm_resume(hba);
10735                         if (ret < 0 && ret != -EACCES) {
10736                                 ufshcd_rpm_put(hba);
10737                                 return ret;
10738                         }
10739                 }
10740                 hba->complete_put = true;
10741         }
10742         return 0;
10743 }
10744 EXPORT_SYMBOL_GPL(__ufshcd_suspend_prepare);
10745
10746 int ufshcd_suspend_prepare(struct device *dev)
10747 {
10748         return __ufshcd_suspend_prepare(dev, true);
10749 }
10750 EXPORT_SYMBOL_GPL(ufshcd_suspend_prepare);
10751
10752 #ifdef CONFIG_PM_SLEEP
10753 static int ufshcd_wl_poweroff(struct device *dev)
10754 {
10755         struct scsi_device *sdev = to_scsi_device(dev);
10756         struct ufs_hba *hba = shost_priv(sdev->host);
10757
10758         __ufshcd_wl_suspend(hba, UFS_SHUTDOWN_PM);
10759         return 0;
10760 }
10761 #endif
10762
10763 static int ufshcd_wl_probe(struct device *dev)
10764 {
10765         struct scsi_device *sdev = to_scsi_device(dev);
10766
10767         if (!is_device_wlun(sdev))
10768                 return -ENODEV;
10769
10770         blk_pm_runtime_init(sdev->request_queue, dev);
10771         pm_runtime_set_autosuspend_delay(dev, 0);
10772         pm_runtime_allow(dev);
10773
10774         return  0;
10775 }
10776
10777 static int ufshcd_wl_remove(struct device *dev)
10778 {
10779         pm_runtime_forbid(dev);
10780         return 0;
10781 }
10782
10783 static const struct dev_pm_ops ufshcd_wl_pm_ops = {
10784 #ifdef CONFIG_PM_SLEEP
10785         .suspend = ufshcd_wl_suspend,
10786         .resume = ufshcd_wl_resume,
10787         .freeze = ufshcd_wl_suspend,
10788         .thaw = ufshcd_wl_resume,
10789         .poweroff = ufshcd_wl_poweroff,
10790         .restore = ufshcd_wl_resume,
10791 #endif
10792         SET_RUNTIME_PM_OPS(ufshcd_wl_runtime_suspend, ufshcd_wl_runtime_resume, NULL)
10793 };
10794
10795 static void ufshcd_check_header_layout(void)
10796 {
10797         /*
10798          * gcc compilers before version 10 cannot do constant-folding for
10799          * sub-byte bitfields. Hence skip the layout checks for gcc 9 and
10800          * before.
10801          */
10802         if (IS_ENABLED(CONFIG_CC_IS_GCC) && CONFIG_GCC_VERSION < 100000)
10803                 return;
10804
10805         BUILD_BUG_ON(((u8 *)&(struct request_desc_header){
10806                                 .cci = 3})[0] != 3);
10807
10808         BUILD_BUG_ON(((u8 *)&(struct request_desc_header){
10809                                 .ehs_length = 2})[1] != 2);
10810
10811         BUILD_BUG_ON(((u8 *)&(struct request_desc_header){
10812                                 .enable_crypto = 1})[2]
10813                      != 0x80);
10814
10815         BUILD_BUG_ON((((u8 *)&(struct request_desc_header){
10816                                         .command_type = 5,
10817                                         .data_direction = 3,
10818                                         .interrupt = 1,
10819                                 })[3]) != ((5 << 4) | (3 << 1) | 1));
10820
10821         BUILD_BUG_ON(((__le32 *)&(struct request_desc_header){
10822                                 .dunl = cpu_to_le32(0xdeadbeef)})[1] !=
10823                 cpu_to_le32(0xdeadbeef));
10824
10825         BUILD_BUG_ON(((u8 *)&(struct request_desc_header){
10826                                 .ocs = 4})[8] != 4);
10827
10828         BUILD_BUG_ON(((u8 *)&(struct request_desc_header){
10829                                 .cds = 5})[9] != 5);
10830
10831         BUILD_BUG_ON(((__le32 *)&(struct request_desc_header){
10832                                 .dunu = cpu_to_le32(0xbadcafe)})[3] !=
10833                 cpu_to_le32(0xbadcafe));
10834
10835         BUILD_BUG_ON(((u8 *)&(struct utp_upiu_header){
10836                              .iid = 0xf })[4] != 0xf0);
10837
10838         BUILD_BUG_ON(((u8 *)&(struct utp_upiu_header){
10839                              .command_set_type = 0xf })[4] != 0xf);
10840 }
10841
10842 /*
10843  * ufs_dev_wlun_template - describes ufs device wlun
10844  * ufs-device wlun - used to send pm commands
10845  * All luns are consumers of ufs-device wlun.
10846  *
10847  * Currently, no sd driver is present for wluns.
10848  * Hence the no specific pm operations are performed.
10849  * With ufs design, SSU should be sent to ufs-device wlun.
10850  * Hence register a scsi driver for ufs wluns only.
10851  */
10852 static struct scsi_driver ufs_dev_wlun_template = {
10853         .gendrv = {
10854                 .name = "ufs_device_wlun",
10855                 .probe = ufshcd_wl_probe,
10856                 .remove = ufshcd_wl_remove,
10857                 .pm = &ufshcd_wl_pm_ops,
10858                 .shutdown = ufshcd_wl_shutdown,
10859         },
10860 };
10861
10862 static int __init ufshcd_core_init(void)
10863 {
10864         int ret;
10865
10866         ufshcd_check_header_layout();
10867
10868         ufs_debugfs_init();
10869
10870         ret = scsi_register_driver(&ufs_dev_wlun_template.gendrv);
10871         if (ret)
10872                 ufs_debugfs_exit();
10873         return ret;
10874 }
10875
10876 static void __exit ufshcd_core_exit(void)
10877 {
10878         ufs_debugfs_exit();
10879         scsi_unregister_driver(&ufs_dev_wlun_template.gendrv);
10880 }
10881
10882 module_init(ufshcd_core_init);
10883 module_exit(ufshcd_core_exit);
10884
10885 MODULE_AUTHOR("Santosh Yaragnavi <[email protected]>");
10886 MODULE_AUTHOR("Vinayak Holikatti <[email protected]>");
10887 MODULE_DESCRIPTION("Generic UFS host controller driver Core");
10888 MODULE_SOFTDEP("pre: governor_simpleondemand");
10889 MODULE_LICENSE("GPL");