drivers/ufs/ufs.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /**
   3  * ufs.c - Universal Flash Storage (UFS) driver
   4  *
   5  * Taken from Linux Kernel v5.2 (drivers/scsi/ufs/ufshcd.c) and ported
   6  * to u-boot.
   7  *
   8  * Copyright (C) 2019 Texas Instruments Incorporated - https://www.ti.com
   9  */
  10
  11 #include <bouncebuf.h>
  12 #include <charset.h>
  13 #include <dm.h>
  14 #include <log.h>
  15 #include <dm/device_compat.h>
  16 #include <dm/devres.h>
  17 #include <dm/lists.h>
  18 #include <dm/device-internal.h>
  19 #include <malloc.h>
  20 #include <hexdump.h>
  21 #include <scsi.h>
  22 #include <asm/io.h>
  23 #include <asm/dma-mapping.h>
  24 #include <linux/bitops.h>
  25 #include <linux/delay.h>
  26 #include <linux/dma-mapping.h>
  27
  28 #include "ufs.h"
  29
  30 #define UFSHCD_ENABLE_INTRS     (UTP_TRANSFER_REQ_COMPL |\
  31                                  UTP_TASK_REQ_COMPL |\
  32                                  UFSHCD_ERROR_MASK)
  33 /* maximum number of link-startup retries */
  34 #define DME_LINKSTARTUP_RETRIES 3
  35
  36 /* maximum number of retries for a general UIC command  */
  37 #define UFS_UIC_COMMAND_RETRIES 3
  38
  39 /* Query request retries */
  40 #define QUERY_REQ_RETRIES 3
  41 /* Query request timeout */
  42 #define QUERY_REQ_TIMEOUT 1500 /* 1.5 seconds */
  43
  44 /* maximum timeout in ms for a general UIC command */
  45 #define UFS_UIC_CMD_TIMEOUT     1000
  46 /* NOP OUT retries waiting for NOP IN response */
  47 #define NOP_OUT_RETRIES    10
  48 /* Timeout after 30 msecs if NOP OUT hangs without response */
  49 #define NOP_OUT_TIMEOUT    30 /* msecs */
  50
  51 /* Only use one Task Tag for all requests */
  52 #define TASK_TAG        0
  53
  54 /* Expose the flag value from utp_upiu_query.value */
  55 #define MASK_QUERY_UPIU_FLAG_LOC 0xFF
  56
  57 #define MAX_PRDT_ENTRY  262144
  58
  59 /* maximum bytes per request */
  60 #define UFS_MAX_BYTES   (128 * 256 * 1024)
  61
  62 static inline bool ufshcd_is_hba_active(struct ufs_hba *hba);
  63 static inline void ufshcd_hba_stop(struct ufs_hba *hba);
  64 static int ufshcd_hba_enable(struct ufs_hba *hba);
  65
  66 /*
  67  * ufshcd_wait_for_register - wait for register value to change
  68  */
  69 static int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask,
  70                                     u32 val, unsigned long timeout_ms)
  71 {
  72         int err = 0;
  73         unsigned long start = get_timer(0);
  74
  75         /* ignore bits that we don't intend to wait on */
  76         val = val & mask;
  77
  78         while ((ufshcd_readl(hba, reg) & mask) != val) {
  79                 if (get_timer(start) > timeout_ms) {
  80                         if ((ufshcd_readl(hba, reg) & mask) != val)
  81                                 err = -ETIMEDOUT;
  82                         break;
  83                 }
  84         }
  85
  86         return err;
  87 }
  88
  89 /**
  90  * ufshcd_init_pwr_info - setting the POR (power on reset)
  91  * values in hba power info
  92  */
  93 static void ufshcd_init_pwr_info(struct ufs_hba *hba)
  94 {
  95         hba->pwr_info.gear_rx = UFS_PWM_G1;
  96         hba->pwr_info.gear_tx = UFS_PWM_G1;
  97         hba->pwr_info.lane_rx = 1;
  98         hba->pwr_info.lane_tx = 1;
  99         hba->pwr_info.pwr_rx = SLOWAUTO_MODE;
 100         hba->pwr_info.pwr_tx = SLOWAUTO_MODE;
 101         hba->pwr_info.hs_rate = 0;
 102 }
 103
 104 /**
 105  * ufshcd_print_pwr_info - print power params as saved in hba
 106  * power info
 107  */
 108 static void ufshcd_print_pwr_info(struct ufs_hba *hba)
 109 {
 110         static const char * const names[] = {
 111                 "INVALID MODE",
 112                 "FAST MODE",
 113                 "SLOW_MODE",
 114                 "INVALID MODE",
 115                 "FASTAUTO_MODE",
 116                 "SLOWAUTO_MODE",
 117                 "INVALID MODE",
 118         };
 119
 120         dev_err(hba->dev, "[RX, TX]: gear=[%d, %d], lane[%d, %d], pwr[%s, %s], rate = %d\n",
 121                 hba->pwr_info.gear_rx, hba->pwr_info.gear_tx,
 122                 hba->pwr_info.lane_rx, hba->pwr_info.lane_tx,
 123                 names[hba->pwr_info.pwr_rx],
 124                 names[hba->pwr_info.pwr_tx],
 125                 hba->pwr_info.hs_rate);
 126 }
 127
 128 /**
 129  * ufshcd_ready_for_uic_cmd - Check if controller is ready
 130  *                            to accept UIC commands
 131  */
 132 static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba)
 133 {
 134         if (ufshcd_readl(hba, REG_CONTROLLER_STATUS) & UIC_COMMAND_READY)
 135                 return true;
 136         else
 137                 return false;
 138 }
 139
 140 /**
 141  * ufshcd_get_uic_cmd_result - Get the UIC command result
 142  */
 143 static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba)
 144 {
 145         return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2) &
 146                MASK_UIC_COMMAND_RESULT;
 147 }
 148
 149 /**
 150  * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command
 151  */
 152 static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba)
 153 {
 154         return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3);
 155 }
 156
 157 /**
 158  * ufshcd_is_device_present - Check if any device connected to
 159  *                            the host controller
 160  */
 161 static inline bool ufshcd_is_device_present(struct ufs_hba *hba)
 162 {
 163         return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) &
 164                                                 DEVICE_PRESENT) ? true : false;
 165 }
 166
 167 /**
 168  * ufshcd_send_uic_cmd - UFS Interconnect layer command API
 169  *
 170  */
 171 static int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
 172 {
 173         unsigned long start = 0;
 174         u32 intr_status;
 175         u32 enabled_intr_status;
 176
 177         if (!ufshcd_ready_for_uic_cmd(hba)) {
 178                 dev_err(hba->dev,
 179                         "Controller not ready to accept UIC commands\n");
 180                 return -EIO;
 181         }
 182
 183         debug("sending uic command:%d\n", uic_cmd->command);
 184
 185         /* Write Args */
 186         ufshcd_writel(hba, uic_cmd->argument1, REG_UIC_COMMAND_ARG_1);
 187         ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2);
 188         ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3);
 189
 190         /* Write UIC Cmd */
 191         ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK,
 192                       REG_UIC_COMMAND);
 193
 194         start = get_timer(0);
 195         do {
 196                 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
 197                 enabled_intr_status = intr_status & hba->intr_mask;
 198                 ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS);
 199
 200                 if (get_timer(start) > UFS_UIC_CMD_TIMEOUT) {
 201                         dev_err(hba->dev,
 202                                 "Timedout waiting for UIC response\n");
 203
 204                         return -ETIMEDOUT;
 205                 }
 206
 207                 if (enabled_intr_status & UFSHCD_ERROR_MASK) {
 208                         dev_err(hba->dev, "Error in status:%08x\n",
 209                                 enabled_intr_status);
 210
 211                         return -1;
 212                 }
 213         } while (!(enabled_intr_status & UFSHCD_UIC_MASK));
 214
 215         uic_cmd->argument2 = ufshcd_get_uic_cmd_result(hba);
 216         uic_cmd->argument3 = ufshcd_get_dme_attr_val(hba);
 217
 218         debug("Sent successfully\n");
 219
 220         return 0;
 221 }
 222
 223 /**
 224  * ufshcd_dme_set_attr - UIC command for DME_SET, DME_PEER_SET
 225  *
 226  */
 227 int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel, u8 attr_set,
 228                         u32 mib_val, u8 peer)
 229 {
 230         struct uic_command uic_cmd = {0};
 231         static const char *const action[] = {
 232                 "dme-set",
 233                 "dme-peer-set"
 234         };
 235         const char *set = action[!!peer];
 236         int ret;
 237         int retries = UFS_UIC_COMMAND_RETRIES;
 238
 239         uic_cmd.command = peer ?
 240                 UIC_CMD_DME_PEER_SET : UIC_CMD_DME_SET;
 241         uic_cmd.argument1 = attr_sel;
 242         uic_cmd.argument2 = UIC_ARG_ATTR_TYPE(attr_set);
 243         uic_cmd.argument3 = mib_val;
 244
 245         do {
 246                 /* for peer attributes we retry upon failure */
 247                 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
 248                 if (ret)
 249                         dev_dbg(hba->dev, "%s: attr-id 0x%x val 0x%x error code %d\n",
 250                                 set, UIC_GET_ATTR_ID(attr_sel), mib_val, ret);
 251         } while (ret && peer && --retries);
 252
 253         if (ret)
 254                 dev_err(hba->dev, "%s: attr-id 0x%x val 0x%x failed %d retries\n",
 255                         set, UIC_GET_ATTR_ID(attr_sel), mib_val,
 256                         UFS_UIC_COMMAND_RETRIES - retries);
 257
 258         return ret;
 259 }
 260
 261 /**
 262  * ufshcd_dme_get_attr - UIC command for DME_GET, DME_PEER_GET
 263  *
 264  */
 265 int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel,
 266                         u32 *mib_val, u8 peer)
 267 {
 268         struct uic_command uic_cmd = {0};
 269         static const char *const action[] = {
 270                 "dme-get",
 271                 "dme-peer-get"
 272         };
 273         const char *get = action[!!peer];
 274         int ret;
 275         int retries = UFS_UIC_COMMAND_RETRIES;
 276
 277         uic_cmd.command = peer ?
 278                 UIC_CMD_DME_PEER_GET : UIC_CMD_DME_GET;
 279         uic_cmd.argument1 = attr_sel;
 280
 281         do {
 282                 /* for peer attributes we retry upon failure */
 283                 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
 284                 if (ret)
 285                         dev_dbg(hba->dev, "%s: attr-id 0x%x error code %d\n",
 286                                 get, UIC_GET_ATTR_ID(attr_sel), ret);
 287         } while (ret && peer && --retries);
 288
 289         if (ret)
 290                 dev_err(hba->dev, "%s: attr-id 0x%x failed %d retries\n",
 291                         get, UIC_GET_ATTR_ID(attr_sel),
 292                         UFS_UIC_COMMAND_RETRIES - retries);
 293
 294         if (mib_val && !ret)
 295                 *mib_val = uic_cmd.argument3;
 296
 297         return ret;
 298 }
 299
 300 static int ufshcd_disable_tx_lcc(struct ufs_hba *hba, bool peer)
 301 {
 302         u32 tx_lanes, i, err = 0;
 303
 304         if (!peer)
 305                 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
 306                                &tx_lanes);
 307         else
 308                 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
 309                                     &tx_lanes);
 310         for (i = 0; i < tx_lanes; i++) {
 311                 if (!peer)
 312                         err = ufshcd_dme_set(hba,
 313                                              UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
 314                                              UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
 315                                              0);
 316                 else
 317                         err = ufshcd_dme_peer_set(hba,
 318                                         UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
 319                                         UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
 320                                         0);
 321                 if (err) {
 322                         dev_err(hba->dev, "%s: TX LCC Disable failed, peer = %d, lane = %d, err = %d\n",
 323                                 __func__, peer, i, err);
 324                         break;
 325                 }
 326         }
 327
 328         return err;
 329 }
 330
 331 static inline int ufshcd_disable_device_tx_lcc(struct ufs_hba *hba)
 332 {
 333         return ufshcd_disable_tx_lcc(hba, true);
 334 }
 335
 336 /**
 337  * ufshcd_dme_link_startup - Notify Unipro to perform link startup
 338  *
 339  */
 340 static int ufshcd_dme_link_startup(struct ufs_hba *hba)
 341 {
 342         struct uic_command uic_cmd = {0};
 343         int ret;
 344
 345         uic_cmd.command = UIC_CMD_DME_LINK_STARTUP;
 346
 347         ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
 348         if (ret)
 349                 dev_dbg(hba->dev,
 350                         "dme-link-startup: error code %d\n", ret);
 351         return ret;
 352 }
 353
 354 /**
 355  * ufshcd_disable_intr_aggr - Disables interrupt aggregation.
 356  *
 357  */
 358 static inline void ufshcd_disable_intr_aggr(struct ufs_hba *hba)
 359 {
 360         ufshcd_writel(hba, 0, REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
 361 }
 362
 363 /**
 364  * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY
 365  */
 366 static inline int ufshcd_get_lists_status(u32 reg)
 367 {
 368         return !((reg & UFSHCD_STATUS_READY) == UFSHCD_STATUS_READY);
 369 }
 370
 371 /**
 372  * ufshcd_enable_run_stop_reg - Enable run-stop registers,
 373  *                      When run-stop registers are set to 1, it indicates the
 374  *                      host controller that it can process the requests
 375  */
 376 static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba)
 377 {
 378         ufshcd_writel(hba, UTP_TASK_REQ_LIST_RUN_STOP_BIT,
 379                       REG_UTP_TASK_REQ_LIST_RUN_STOP);
 380         ufshcd_writel(hba, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT,
 381                       REG_UTP_TRANSFER_REQ_LIST_RUN_STOP);
 382 }
 383
 384 /**
 385  * ufshcd_enable_intr - enable interrupts
 386  */
 387 static void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs)
 388 {
 389         u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
 390         u32 rw;
 391
 392         if (hba->version == UFSHCI_VERSION_10) {
 393                 rw = set & INTERRUPT_MASK_RW_VER_10;
 394                 set = rw | ((set ^ intrs) & intrs);
 395         } else {
 396                 set |= intrs;
 397         }
 398
 399         ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
 400
 401         hba->intr_mask = set;
 402 }
 403
 404 /**
 405  * ufshcd_make_hba_operational - Make UFS controller operational
 406  *
 407  * To bring UFS host controller to operational state,
 408  * 1. Enable required interrupts
 409  * 2. Configure interrupt aggregation
 410  * 3. Program UTRL and UTMRL base address
 411  * 4. Configure run-stop-registers
 412  *
 413  */
 414 static int ufshcd_make_hba_operational(struct ufs_hba *hba)
 415 {
 416         int err = 0;
 417         u32 reg;
 418
 419         /* Enable required interrupts */
 420         ufshcd_enable_intr(hba, UFSHCD_ENABLE_INTRS);
 421
 422         /* Disable interrupt aggregation */
 423         ufshcd_disable_intr_aggr(hba);
 424
 425         /* Configure UTRL and UTMRL base address registers */
 426         ufshcd_writel(hba, lower_32_bits((dma_addr_t)hba->utrdl),
 427                       REG_UTP_TRANSFER_REQ_LIST_BASE_L);
 428         ufshcd_writel(hba, upper_32_bits((dma_addr_t)hba->utrdl),
 429                       REG_UTP_TRANSFER_REQ_LIST_BASE_H);
 430         ufshcd_writel(hba, lower_32_bits((dma_addr_t)hba->utmrdl),
 431                       REG_UTP_TASK_REQ_LIST_BASE_L);
 432         ufshcd_writel(hba, upper_32_bits((dma_addr_t)hba->utmrdl),
 433                       REG_UTP_TASK_REQ_LIST_BASE_H);
 434
 435         /*
 436          * UCRDY, UTMRLDY and UTRLRDY bits must be 1
 437          */
 438         reg = ufshcd_readl(hba, REG_CONTROLLER_STATUS);
 439         if (!(ufshcd_get_lists_status(reg))) {
 440                 ufshcd_enable_run_stop_reg(hba);
 441         } else {
 442                 dev_err(hba->dev,
 443                         "Host controller not ready to process requests\n");
 444                 err = -EIO;
 445                 goto out;
 446         }
 447
 448 out:
 449         return err;
 450 }
 451
 452 /**
 453  * ufshcd_link_startup - Initialize unipro link startup
 454  */
 455 static int ufshcd_link_startup(struct ufs_hba *hba)
 456 {
 457         int ret;
 458         int retries = DME_LINKSTARTUP_RETRIES;
 459         bool link_startup_again = true;
 460
 461 link_startup:
 462         do {
 463                 ufshcd_ops_link_startup_notify(hba, PRE_CHANGE);
 464
 465                 ret = ufshcd_dme_link_startup(hba);
 466
 467                 /* check if device is detected by inter-connect layer */
 468                 if (!ret && !ufshcd_is_device_present(hba)) {
 469                         dev_err(hba->dev, "%s: Device not present\n", __func__);
 470                         ret = -ENXIO;
 471                         goto out;
 472                 }
 473
 474                 /*
 475                  * DME link lost indication is only received when link is up,
 476                  * but we can't be sure if the link is up until link startup
 477                  * succeeds. So reset the local Uni-Pro and try again.
 478                  */
 479                 if (ret && ufshcd_hba_enable(hba))
 480                         goto out;
 481         } while (ret && retries--);
 482
 483         if (ret)
 484                 /* failed to get the link up... retire */
 485                 goto out;
 486
 487         if (link_startup_again) {
 488                 link_startup_again = false;
 489                 retries = DME_LINKSTARTUP_RETRIES;
 490                 goto link_startup;
 491         }
 492
 493         /* Mark that link is up in PWM-G1, 1-lane, SLOW-AUTO mode */
 494         ufshcd_init_pwr_info(hba);
 495
 496         if (hba->quirks & UFSHCD_QUIRK_BROKEN_LCC) {
 497                 ret = ufshcd_disable_device_tx_lcc(hba);
 498                 if (ret)
 499                         goto out;
 500         }
 501
 502         /* Include any host controller configuration via UIC commands */
 503         ret = ufshcd_ops_link_startup_notify(hba, POST_CHANGE);
 504         if (ret)
 505                 goto out;
 506
 507         ret = ufshcd_make_hba_operational(hba);
 508 out:
 509         if (ret)
 510                 dev_err(hba->dev, "link startup failed %d\n", ret);
 511
 512         return ret;
 513 }
 514
 515 /**
 516  * ufshcd_hba_stop - Send controller to reset state
 517  */
 518 static inline void ufshcd_hba_stop(struct ufs_hba *hba)
 519 {
 520         int err;
 521
 522         ufshcd_writel(hba, CONTROLLER_DISABLE,  REG_CONTROLLER_ENABLE);
 523         err = ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE,
 524                                        CONTROLLER_ENABLE, CONTROLLER_DISABLE,
 525                                        10);
 526         if (err)
 527                 dev_err(hba->dev, "%s: Controller disable failed\n", __func__);
 528 }
 529
 530 /**
 531  * ufshcd_is_hba_active - Get controller state
 532  */
 533 static inline bool ufshcd_is_hba_active(struct ufs_hba *hba)
 534 {
 535         return (ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & CONTROLLER_ENABLE)
 536                 ? false : true;
 537 }
 538
 539 /**
 540  * ufshcd_hba_start - Start controller initialization sequence
 541  */
 542 static inline void ufshcd_hba_start(struct ufs_hba *hba)
 543 {
 544         ufshcd_writel(hba, CONTROLLER_ENABLE, REG_CONTROLLER_ENABLE);
 545 }
 546
 547 /**
 548  * ufshcd_hba_enable - initialize the controller
 549  */
 550 static int ufshcd_hba_enable(struct ufs_hba *hba)
 551 {
 552         int retry;
 553
 554         if (!ufshcd_is_hba_active(hba))
 555                 /* change controller state to "reset state" */
 556                 ufshcd_hba_stop(hba);
 557
 558         ufshcd_ops_hce_enable_notify(hba, PRE_CHANGE);
 559
 560         /* start controller initialization sequence */
 561         ufshcd_hba_start(hba);
 562
 563         /*
 564          * To initialize a UFS host controller HCE bit must be set to 1.
 565          * During initialization the HCE bit value changes from 1->0->1.
 566          * When the host controller completes initialization sequence
 567          * it sets the value of HCE bit to 1. The same HCE bit is read back
 568          * to check if the controller has completed initialization sequence.
 569          * So without this delay the value HCE = 1, set in the previous
 570          * instruction might be read back.
 571          * This delay can be changed based on the controller.
 572          */
 573         mdelay(1);
 574
 575         /* wait for the host controller to complete initialization */
 576         retry = 10;
 577         while (ufshcd_is_hba_active(hba)) {
 578                 if (retry) {
 579                         retry--;
 580                 } else {
 581                         dev_err(hba->dev, "Controller enable failed\n");
 582                         return -EIO;
 583                 }
 584                 mdelay(5);
 585         }
 586
 587         /* enable UIC related interrupts */
 588         ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
 589
 590         ufshcd_ops_hce_enable_notify(hba, POST_CHANGE);
 591
 592         return 0;
 593 }
 594
 595 /**
 596  * ufshcd_host_memory_configure - configure local reference block with
 597  *                              memory offsets
 598  */
 599 static void ufshcd_host_memory_configure(struct ufs_hba *hba)
 600 {
 601         struct utp_transfer_req_desc *utrdlp;
 602         dma_addr_t cmd_desc_dma_addr;
 603         u16 response_offset;
 604         u16 prdt_offset;
 605
 606         utrdlp = hba->utrdl;
 607         cmd_desc_dma_addr = (dma_addr_t)hba->ucdl;
 608
 609         utrdlp->command_desc_base_addr_lo =
 610                                 cpu_to_le32(lower_32_bits(cmd_desc_dma_addr));
 611         utrdlp->command_desc_base_addr_hi =
 612                                 cpu_to_le32(upper_32_bits(cmd_desc_dma_addr));
 613
 614         response_offset = offsetof(struct utp_transfer_cmd_desc, response_upiu);
 615         prdt_offset = offsetof(struct utp_transfer_cmd_desc, prd_table);
 616
 617         utrdlp->response_upiu_offset = cpu_to_le16(response_offset >> 2);
 618         utrdlp->prd_table_offset = cpu_to_le16(prdt_offset >> 2);
 619         utrdlp->response_upiu_length = cpu_to_le16(ALIGNED_UPIU_SIZE >> 2);
 620
 621         hba->ucd_req_ptr = (struct utp_upiu_req *)hba->ucdl;
 622         hba->ucd_rsp_ptr =
 623                 (struct utp_upiu_rsp *)&hba->ucdl->response_upiu;
 624         hba->ucd_prdt_ptr =
 625                 (struct ufshcd_sg_entry *)&hba->ucdl->prd_table;
 626 }
 627
 628 /**
 629  * ufshcd_memory_alloc - allocate memory for host memory space data structures
 630  */
 631 static int ufshcd_memory_alloc(struct ufs_hba *hba)
 632 {
 633         /* Allocate one Transfer Request Descriptor
 634          * Should be aligned to 1k boundary.
 635          */
 636         hba->utrdl = memalign(1024, sizeof(struct utp_transfer_req_desc));
 637         if (!hba->utrdl) {
 638                 dev_err(hba->dev, "Transfer Descriptor memory allocation failed\n");
 639                 return -ENOMEM;
 640         }
 641
 642         /* Allocate one Command Descriptor
 643          * Should be aligned to 1k boundary.
 644          */
 645         hba->ucdl = memalign(1024, sizeof(struct utp_transfer_cmd_desc));
 646         if (!hba->ucdl) {
 647                 dev_err(hba->dev, "Command descriptor memory allocation failed\n");
 648                 return -ENOMEM;
 649         }
 650
 651         return 0;
 652 }
 653
 654 /**
 655  * ufshcd_get_intr_mask - Get the interrupt bit mask
 656  */
 657 static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba)
 658 {
 659         u32 intr_mask = 0;
 660
 661         switch (hba->version) {
 662         case UFSHCI_VERSION_10:
 663                 intr_mask = INTERRUPT_MASK_ALL_VER_10;
 664                 break;
 665         case UFSHCI_VERSION_11:
 666         case UFSHCI_VERSION_20:
 667                 intr_mask = INTERRUPT_MASK_ALL_VER_11;
 668                 break;
 669         case UFSHCI_VERSION_21:
 670         default:
 671                 intr_mask = INTERRUPT_MASK_ALL_VER_21;
 672                 break;
 673         }
 674
 675         return intr_mask;
 676 }
 677
 678 /**
 679  * ufshcd_get_ufs_version - Get the UFS version supported by the HBA
 680  */
 681 static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba)
 682 {
 683         return ufshcd_readl(hba, REG_UFS_VERSION);
 684 }
 685
 686 /**
 687  * ufshcd_get_upmcrs - Get the power mode change request status
 688  */
 689 static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba)
 690 {
 691         return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) >> 8) & 0x7;
 692 }
 693
 694 /**
 695  * ufshcd_cache_flush_and_invalidate - Flush and invalidate cache
 696  *
 697  * Flush and invalidate cache in aligned address..address+size range.
 698  * The invalidation is in place to avoid stale data in cache.
 699  */
 700 static void ufshcd_cache_flush_and_invalidate(void *addr, unsigned long size)
 701 {
 702         uintptr_t aaddr = (uintptr_t)addr & ~(ARCH_DMA_MINALIGN - 1);
 703         unsigned long asize = ALIGN(size, ARCH_DMA_MINALIGN);
 704
 705         flush_dcache_range(aaddr, aaddr + asize);
 706         invalidate_dcache_range(aaddr, aaddr + asize);
 707 }
 708
 709 /**
 710  * ufshcd_prepare_req_desc_hdr() - Fills the requests header
 711  * descriptor according to request
 712  */
 713 static void ufshcd_prepare_req_desc_hdr(struct ufs_hba *hba,
 714                                         u32 *upiu_flags,
 715                                         enum dma_data_direction cmd_dir)
 716 {
 717         struct utp_transfer_req_desc *req_desc = hba->utrdl;
 718         u32 data_direction;
 719         u32 dword_0;
 720
 721         if (cmd_dir == DMA_FROM_DEVICE) {
 722                 data_direction = UTP_DEVICE_TO_HOST;
 723                 *upiu_flags = UPIU_CMD_FLAGS_READ;
 724         } else if (cmd_dir == DMA_TO_DEVICE) {
 725                 data_direction = UTP_HOST_TO_DEVICE;
 726                 *upiu_flags = UPIU_CMD_FLAGS_WRITE;
 727         } else {
 728                 data_direction = UTP_NO_DATA_TRANSFER;
 729                 *upiu_flags = UPIU_CMD_FLAGS_NONE;
 730         }
 731
 732         dword_0 = data_direction | (0x1 << UPIU_COMMAND_TYPE_OFFSET);
 733
 734         /* Enable Interrupt for command */
 735         dword_0 |= UTP_REQ_DESC_INT_CMD;
 736
 737         /* Transfer request descriptor header fields */
 738         req_desc->header.dword_0 = cpu_to_le32(dword_0);
 739         /* dword_1 is reserved, hence it is set to 0 */
 740         req_desc->header.dword_1 = 0;
 741         /*
 742          * assigning invalid value for command status. Controller
 743          * updates OCS on command completion, with the command
 744          * status
 745          */
 746         req_desc->header.dword_2 =
 747                 cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
 748         /* dword_3 is reserved, hence it is set to 0 */
 749         req_desc->header.dword_3 = 0;
 750
 751         req_desc->prd_table_length = 0;
 752
 753         ufshcd_cache_flush_and_invalidate(req_desc, sizeof(*req_desc));
 754 }
 755
 756 static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba,
 757                                               u32 upiu_flags)
 758 {
 759         struct utp_upiu_req *ucd_req_ptr = hba->ucd_req_ptr;
 760         struct ufs_query *query = &hba->dev_cmd.query;
 761         u16 len = be16_to_cpu(query->request.upiu_req.length);
 762
 763         /* Query request header */
 764         ucd_req_ptr->header.dword_0 =
 765                                 UPIU_HEADER_DWORD(UPIU_TRANSACTION_QUERY_REQ,
 766                                                   upiu_flags, 0, TASK_TAG);
 767         ucd_req_ptr->header.dword_1 =
 768                                 UPIU_HEADER_DWORD(0, query->request.query_func,
 769                                                   0, 0);
 770
 771         /* Data segment length only need for WRITE_DESC */
 772         if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
 773                 ucd_req_ptr->header.dword_2 =
 774                                 UPIU_HEADER_DWORD(0, 0, (len >> 8), (u8)len);
 775         else
 776                 ucd_req_ptr->header.dword_2 = 0;
 777
 778         /* Copy the Query Request buffer as is */
 779         memcpy(&ucd_req_ptr->qr, &query->request.upiu_req, QUERY_OSF_SIZE);
 780
 781         /* Copy the Descriptor */
 782         if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC) {
 783                 memcpy(ucd_req_ptr + 1, query->descriptor, len);
 784                 ufshcd_cache_flush_and_invalidate(ucd_req_ptr, 2 * sizeof(*ucd_req_ptr));
 785         } else {
 786                 ufshcd_cache_flush_and_invalidate(ucd_req_ptr, sizeof(*ucd_req_ptr));
 787         }
 788
 789         memset(hba->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
 790         ufshcd_cache_flush_and_invalidate(hba->ucd_rsp_ptr, sizeof(*hba->ucd_rsp_ptr));
 791 }
 792
 793 static inline void ufshcd_prepare_utp_nop_upiu(struct ufs_hba *hba)
 794 {
 795         struct utp_upiu_req *ucd_req_ptr = hba->ucd_req_ptr;
 796
 797         memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req));
 798
 799         /* command descriptor fields */
 800         ucd_req_ptr->header.dword_0 =
 801                         UPIU_HEADER_DWORD(UPIU_TRANSACTION_NOP_OUT, 0, 0, TASK_TAG);
 802         /* clear rest of the fields of basic header */
 803         ucd_req_ptr->header.dword_1 = 0;
 804         ucd_req_ptr->header.dword_2 = 0;
 805
 806         memset(hba->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
 807
 808         ufshcd_cache_flush_and_invalidate(ucd_req_ptr, sizeof(*ucd_req_ptr));
 809         ufshcd_cache_flush_and_invalidate(hba->ucd_rsp_ptr, sizeof(*hba->ucd_rsp_ptr));
 810 }
 811
 812 /**
 813  * ufshcd_comp_devman_upiu - UFS Protocol Information Unit(UPIU)
 814  *                           for Device Management Purposes
 815  */
 816 static int ufshcd_comp_devman_upiu(struct ufs_hba *hba,
 817                                    enum dev_cmd_type cmd_type)
 818 {
 819         u32 upiu_flags;
 820         int ret = 0;
 821
 822         hba->dev_cmd.type = cmd_type;
 823
 824         ufshcd_prepare_req_desc_hdr(hba, &upiu_flags, DMA_NONE);
 825         switch (cmd_type) {
 826         case DEV_CMD_TYPE_QUERY:
 827                 ufshcd_prepare_utp_query_req_upiu(hba, upiu_flags);
 828                 break;
 829         case DEV_CMD_TYPE_NOP:
 830                 ufshcd_prepare_utp_nop_upiu(hba);
 831                 break;
 832         default:
 833                 ret = -EINVAL;
 834         }
 835
 836         return ret;
 837 }
 838
 839 static int ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag)
 840 {
 841         unsigned long start;
 842         u32 intr_status;
 843         u32 enabled_intr_status;
 844
 845         ufshcd_writel(hba, 1 << task_tag, REG_UTP_TRANSFER_REQ_DOOR_BELL);
 846
 847         start = get_timer(0);
 848         do {
 849                 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
 850                 enabled_intr_status = intr_status & hba->intr_mask;
 851                 ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS);
 852
 853                 if (get_timer(start) > QUERY_REQ_TIMEOUT) {
 854                         dev_err(hba->dev,
 855                                 "Timedout waiting for UTP response\n");
 856
 857                         return -ETIMEDOUT;
 858                 }
 859
 860                 if (enabled_intr_status & UFSHCD_ERROR_MASK) {
 861                         dev_err(hba->dev, "Error in status:%08x\n",
 862                                 enabled_intr_status);
 863
 864                         return -1;
 865                 }
 866         } while (!(enabled_intr_status & UTP_TRANSFER_REQ_COMPL));
 867
 868         return 0;
 869 }
 870
 871 /**
 872  * ufshcd_get_req_rsp - returns the TR response transaction type
 873  */
 874 static inline int ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr)
 875 {
 876         return be32_to_cpu(ucd_rsp_ptr->header.dword_0) >> 24;
 877 }
 878
 879 /**
 880  * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status
 881  *
 882  */
 883 static inline int ufshcd_get_tr_ocs(struct ufs_hba *hba)
 884 {
 885         struct utp_transfer_req_desc *req_desc = hba->utrdl;
 886
 887         return le32_to_cpu(req_desc->header.dword_2) & MASK_OCS;
 888 }
 889
 890 static inline int ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp *ucd_rsp_ptr)
 891 {
 892         return be32_to_cpu(ucd_rsp_ptr->header.dword_1) & MASK_RSP_UPIU_RESULT;
 893 }
 894
 895 static int ufshcd_check_query_response(struct ufs_hba *hba)
 896 {
 897         struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
 898
 899         /* Get the UPIU response */
 900         query_res->response = ufshcd_get_rsp_upiu_result(hba->ucd_rsp_ptr) >>
 901                                 UPIU_RSP_CODE_OFFSET;
 902         return query_res->response;
 903 }
 904
 905 /**
 906  * ufshcd_copy_query_response() - Copy the Query Response and the data
 907  * descriptor
 908  */
 909 static int ufshcd_copy_query_response(struct ufs_hba *hba)
 910 {
 911         struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
 912
 913         memcpy(&query_res->upiu_res, &hba->ucd_rsp_ptr->qr, QUERY_OSF_SIZE);
 914
 915         /* Get the descriptor */
 916         if (hba->dev_cmd.query.descriptor &&
 917             hba->ucd_rsp_ptr->qr.opcode == UPIU_QUERY_OPCODE_READ_DESC) {
 918                 u8 *descp = (u8 *)hba->ucd_rsp_ptr +
 919                                 GENERAL_UPIU_REQUEST_SIZE;
 920                 u16 resp_len;
 921                 u16 buf_len;
 922
 923                 /* data segment length */
 924                 resp_len = be32_to_cpu(hba->ucd_rsp_ptr->header.dword_2) &
 925                                                 MASK_QUERY_DATA_SEG_LEN;
 926                 buf_len =
 927                         be16_to_cpu(hba->dev_cmd.query.request.upiu_req.length);
 928                 if (likely(buf_len >= resp_len)) {
 929                         memcpy(hba->dev_cmd.query.descriptor, descp, resp_len);
 930                 } else {
 931                         dev_warn(hba->dev,
 932                                  "%s: Response size is bigger than buffer\n",
 933                                  __func__);
 934                         return -EINVAL;
 935                 }
 936         }
 937
 938         return 0;
 939 }
 940
 941 /**
 942  * ufshcd_exec_dev_cmd - API for sending device management requests
 943  */
 944 static int ufshcd_exec_dev_cmd(struct ufs_hba *hba, enum dev_cmd_type cmd_type,
 945                                int timeout)
 946 {
 947         int err;
 948         int resp;
 949
 950         err = ufshcd_comp_devman_upiu(hba, cmd_type);
 951         if (err)
 952                 return err;
 953
 954         err = ufshcd_send_command(hba, TASK_TAG);
 955         if (err)
 956                 return err;
 957
 958         err = ufshcd_get_tr_ocs(hba);
 959         if (err) {
 960                 dev_err(hba->dev, "Error in OCS:%d\n", err);
 961                 return -EINVAL;
 962         }
 963
 964         resp = ufshcd_get_req_rsp(hba->ucd_rsp_ptr);
 965         switch (resp) {
 966         case UPIU_TRANSACTION_NOP_IN:
 967                 break;
 968         case UPIU_TRANSACTION_QUERY_RSP:
 969                 err = ufshcd_check_query_response(hba);
 970                 if (!err)
 971                         err = ufshcd_copy_query_response(hba);
 972                 break;
 973         case UPIU_TRANSACTION_REJECT_UPIU:
 974                 /* TODO: handle Reject UPIU Response */
 975                 err = -EPERM;
 976                 dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n",
 977                         __func__);
 978                 break;
 979         default:
 980                 err = -EINVAL;
 981                 dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n",
 982                         __func__, resp);
 983         }
 984
 985         return err;
 986 }
 987
 988 /**
 989  * ufshcd_init_query() - init the query response and request parameters
 990  */
 991 static inline void ufshcd_init_query(struct ufs_hba *hba,
 992                                      struct ufs_query_req **request,
 993                                      struct ufs_query_res **response,
 994                                      enum query_opcode opcode,
 995                                      u8 idn, u8 index, u8 selector)
 996 {
 997         *request = &hba->dev_cmd.query.request;
 998         *response = &hba->dev_cmd.query.response;
 999         memset(*request, 0, sizeof(struct ufs_query_req));
1000         memset(*response, 0, sizeof(struct ufs_query_res));
1001         (*request)->upiu_req.opcode = opcode;
1002         (*request)->upiu_req.idn = idn;
1003         (*request)->upiu_req.index = index;
1004         (*request)->upiu_req.selector = selector;
1005 }
1006
1007 /**
1008  * ufshcd_query_flag() - API function for sending flag query requests
1009  */
1010 int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode,
1011                       enum flag_idn idn, bool *flag_res)
1012 {
1013         struct ufs_query_req *request = NULL;
1014         struct ufs_query_res *response = NULL;
1015         int err, index = 0, selector = 0;
1016         int timeout = QUERY_REQ_TIMEOUT;
1017
1018         ufshcd_init_query(hba, &request, &response, opcode, idn, index,
1019                           selector);
1020
1021         switch (opcode) {
1022         case UPIU_QUERY_OPCODE_SET_FLAG:
1023         case UPIU_QUERY_OPCODE_CLEAR_FLAG:
1024         case UPIU_QUERY_OPCODE_TOGGLE_FLAG:
1025                 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
1026                 break;
1027         case UPIU_QUERY_OPCODE_READ_FLAG:
1028                 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
1029                 if (!flag_res) {
1030                         /* No dummy reads */
1031                         dev_err(hba->dev, "%s: Invalid argument for read request\n",
1032                                 __func__);
1033                         err = -EINVAL;
1034                         goto out;
1035                 }
1036                 break;
1037         default:
1038                 dev_err(hba->dev,
1039                         "%s: Expected query flag opcode but got = %d\n",
1040                         __func__, opcode);
1041                 err = -EINVAL;
1042                 goto out;
1043         }
1044
1045         err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, timeout);
1046
1047         if (err) {
1048                 dev_err(hba->dev,
1049                         "%s: Sending flag query for idn %d failed, err = %d\n",
1050                         __func__, idn, err);
1051                 goto out;
1052         }
1053
1054         if (flag_res)
1055                 *flag_res = (be32_to_cpu(response->upiu_res.value) &
1056                                 MASK_QUERY_UPIU_FLAG_LOC) & 0x1;
1057
1058 out:
1059         return err;
1060 }
1061
1062 static int ufshcd_query_flag_retry(struct ufs_hba *hba,
1063                                    enum query_opcode opcode,
1064                                    enum flag_idn idn, bool *flag_res)
1065 {
1066         int ret;
1067         int retries;
1068
1069         for (retries = 0; retries < QUERY_REQ_RETRIES; retries++) {
1070                 ret = ufshcd_query_flag(hba, opcode, idn, flag_res);
1071                 if (ret)
1072                         dev_dbg(hba->dev,
1073                                 "%s: failed with error %d, retries %d\n",
1074                                 __func__, ret, retries);
1075                 else
1076                         break;
1077         }
1078
1079         if (ret)
1080                 dev_err(hba->dev,
1081                         "%s: query attribute, opcode %d, idn %d, failed with error %d after %d retires\n",
1082                         __func__, opcode, idn, ret, retries);
1083         return ret;
1084 }
1085
1086 static int __ufshcd_query_descriptor(struct ufs_hba *hba,
1087                                      enum query_opcode opcode,
1088                                      enum desc_idn idn, u8 index, u8 selector,
1089                                      u8 *desc_buf, int *buf_len)
1090 {
1091         struct ufs_query_req *request = NULL;
1092         struct ufs_query_res *response = NULL;
1093         int err;
1094
1095         if (!desc_buf) {
1096                 dev_err(hba->dev, "%s: descriptor buffer required for opcode 0x%x\n",
1097                         __func__, opcode);
1098                 err = -EINVAL;
1099                 goto out;
1100         }
1101
1102         if (*buf_len < QUERY_DESC_MIN_SIZE || *buf_len > QUERY_DESC_MAX_SIZE) {
1103                 dev_err(hba->dev, "%s: descriptor buffer size (%d) is out of range\n",
1104                         __func__, *buf_len);
1105                 err = -EINVAL;
1106                 goto out;
1107         }
1108
1109         ufshcd_init_query(hba, &request, &response, opcode, idn, index,
1110                           selector);
1111         hba->dev_cmd.query.descriptor = desc_buf;
1112         request->upiu_req.length = cpu_to_be16(*buf_len);
1113
1114         switch (opcode) {
1115         case UPIU_QUERY_OPCODE_WRITE_DESC:
1116                 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
1117                 break;
1118         case UPIU_QUERY_OPCODE_READ_DESC:
1119                 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
1120                 break;
1121         default:
1122                 dev_err(hba->dev, "%s: Expected query descriptor opcode but got = 0x%.2x\n",
1123                         __func__, opcode);
1124                 err = -EINVAL;
1125                 goto out;
1126         }
1127
1128         err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
1129
1130         if (err) {
1131                 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
1132                         __func__, opcode, idn, index, err);
1133                 goto out;
1134         }
1135
1136         hba->dev_cmd.query.descriptor = NULL;
1137         *buf_len = be16_to_cpu(response->upiu_res.length);
1138
1139 out:
1140         return err;
1141 }
1142
1143 /**
1144  * ufshcd_query_descriptor_retry - API function for sending descriptor requests
1145  */
1146 int ufshcd_query_descriptor_retry(struct ufs_hba *hba, enum query_opcode opcode,
1147                                   enum desc_idn idn, u8 index, u8 selector,
1148                                   u8 *desc_buf, int *buf_len)
1149 {
1150         int err;
1151         int retries;
1152
1153         for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
1154                 err = __ufshcd_query_descriptor(hba, opcode, idn, index,
1155                                                 selector, desc_buf, buf_len);
1156                 if (!err || err == -EINVAL)
1157                         break;
1158         }
1159
1160         return err;
1161 }
1162
1163 /**
1164  * ufshcd_read_desc_length - read the specified descriptor length from header
1165  */
1166 static int ufshcd_read_desc_length(struct ufs_hba *hba, enum desc_idn desc_id,
1167                                    int desc_index, int *desc_length)
1168 {
1169         int ret;
1170         u8 header[QUERY_DESC_HDR_SIZE];
1171         int header_len = QUERY_DESC_HDR_SIZE;
1172
1173         if (desc_id >= QUERY_DESC_IDN_MAX)
1174                 return -EINVAL;
1175
1176         ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC,
1177                                             desc_id, desc_index, 0, header,
1178                                             &header_len);
1179
1180         if (ret) {
1181                 dev_err(hba->dev, "%s: Failed to get descriptor header id %d\n",
1182                         __func__, desc_id);
1183                 return ret;
1184         } else if (desc_id != header[QUERY_DESC_DESC_TYPE_OFFSET]) {
1185                 dev_warn(hba->dev, "%s: descriptor header id %d and desc_id %d mismatch\n",
1186                          __func__, header[QUERY_DESC_DESC_TYPE_OFFSET],
1187                          desc_id);
1188                 ret = -EINVAL;
1189         }
1190
1191         *desc_length = header[QUERY_DESC_LENGTH_OFFSET];
1192
1193         return ret;
1194 }
1195
1196 static void ufshcd_init_desc_sizes(struct ufs_hba *hba)
1197 {
1198         int err;
1199
1200         err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_DEVICE, 0,
1201                                       &hba->desc_size.dev_desc);
1202         if (err)
1203                 hba->desc_size.dev_desc = QUERY_DESC_DEVICE_DEF_SIZE;
1204
1205         err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_POWER, 0,
1206                                       &hba->desc_size.pwr_desc);
1207         if (err)
1208                 hba->desc_size.pwr_desc = QUERY_DESC_POWER_DEF_SIZE;
1209
1210         err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_INTERCONNECT, 0,
1211                                       &hba->desc_size.interc_desc);
1212         if (err)
1213                 hba->desc_size.interc_desc = QUERY_DESC_INTERCONNECT_DEF_SIZE;
1214
1215         err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_CONFIGURATION, 0,
1216                                       &hba->desc_size.conf_desc);
1217         if (err)
1218                 hba->desc_size.conf_desc = QUERY_DESC_CONFIGURATION_DEF_SIZE;
1219
1220         err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_UNIT, 0,
1221                                       &hba->desc_size.unit_desc);
1222         if (err)
1223                 hba->desc_size.unit_desc = QUERY_DESC_UNIT_DEF_SIZE;
1224
1225         err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_GEOMETRY, 0,
1226                                       &hba->desc_size.geom_desc);
1227         if (err)
1228                 hba->desc_size.geom_desc = QUERY_DESC_GEOMETRY_DEF_SIZE;
1229
1230         err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_HEALTH, 0,
1231                                       &hba->desc_size.hlth_desc);
1232         if (err)
1233                 hba->desc_size.hlth_desc = QUERY_DESC_HEALTH_DEF_SIZE;
1234 }
1235
1236 /**
1237  * ufshcd_map_desc_id_to_length - map descriptor IDN to its length
1238  *
1239  */
1240 int ufshcd_map_desc_id_to_length(struct ufs_hba *hba, enum desc_idn desc_id,
1241                                  int *desc_len)
1242 {
1243         switch (desc_id) {
1244         case QUERY_DESC_IDN_DEVICE:
1245                 *desc_len = hba->desc_size.dev_desc;
1246                 break;
1247         case QUERY_DESC_IDN_POWER:
1248                 *desc_len = hba->desc_size.pwr_desc;
1249                 break;
1250         case QUERY_DESC_IDN_GEOMETRY:
1251                 *desc_len = hba->desc_size.geom_desc;
1252                 break;
1253         case QUERY_DESC_IDN_CONFIGURATION:
1254                 *desc_len = hba->desc_size.conf_desc;
1255                 break;
1256         case QUERY_DESC_IDN_UNIT:
1257                 *desc_len = hba->desc_size.unit_desc;
1258                 break;
1259         case QUERY_DESC_IDN_INTERCONNECT:
1260                 *desc_len = hba->desc_size.interc_desc;
1261                 break;
1262         case QUERY_DESC_IDN_STRING:
1263                 *desc_len = QUERY_DESC_MAX_SIZE;
1264                 break;
1265         case QUERY_DESC_IDN_HEALTH:
1266                 *desc_len = hba->desc_size.hlth_desc;
1267                 break;
1268         case QUERY_DESC_IDN_RFU_0:
1269         case QUERY_DESC_IDN_RFU_1:
1270                 *desc_len = 0;
1271                 break;
1272         default:
1273                 *desc_len = 0;
1274                 return -EINVAL;
1275         }
1276         return 0;
1277 }
1278 EXPORT_SYMBOL(ufshcd_map_desc_id_to_length);
1279
1280 /**
1281  * ufshcd_read_desc_param - read the specified descriptor parameter
1282  *
1283  */
1284 int ufshcd_read_desc_param(struct ufs_hba *hba, enum desc_idn desc_id,
1285                            int desc_index, u8 param_offset, u8 *param_read_buf,
1286                            u8 param_size)
1287 {
1288         int ret;
1289         u8 *desc_buf;
1290         int buff_len;
1291         bool is_kmalloc = true;
1292
1293         /* Safety check */
1294         if (desc_id >= QUERY_DESC_IDN_MAX || !param_size)
1295                 return -EINVAL;
1296
1297         /* Get the max length of descriptor from structure filled up at probe
1298          * time.
1299          */
1300         ret = ufshcd_map_desc_id_to_length(hba, desc_id, &buff_len);
1301
1302         /* Sanity checks */
1303         if (ret || !buff_len) {
1304                 dev_err(hba->dev, "%s: Failed to get full descriptor length\n",
1305                         __func__);
1306                 return ret;
1307         }
1308
1309         /* Check whether we need temp memory */
1310         if (param_offset != 0 || param_size < buff_len) {
1311                 desc_buf = kmalloc(buff_len, GFP_KERNEL);
1312                 if (!desc_buf)
1313                         return -ENOMEM;
1314         } else {
1315                 desc_buf = param_read_buf;
1316                 is_kmalloc = false;
1317         }
1318
1319         /* Request for full descriptor */
1320         ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC,
1321                                             desc_id, desc_index, 0, desc_buf,
1322                                             &buff_len);
1323
1324         if (ret) {
1325                 dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d, desc_index %d, param_offset %d, ret %d\n",
1326                         __func__, desc_id, desc_index, param_offset, ret);
1327                 goto out;
1328         }
1329
1330         /* Sanity check */
1331         if (desc_buf[QUERY_DESC_DESC_TYPE_OFFSET] != desc_id) {
1332                 dev_err(hba->dev, "%s: invalid desc_id %d in descriptor header\n",
1333                         __func__, desc_buf[QUERY_DESC_DESC_TYPE_OFFSET]);
1334                 ret = -EINVAL;
1335                 goto out;
1336         }
1337
1338         /* Check wherher we will not copy more data, than available */
1339         if (is_kmalloc && param_size > buff_len)
1340                 param_size = buff_len;
1341
1342         if (is_kmalloc)
1343                 memcpy(param_read_buf, &desc_buf[param_offset], param_size);
1344 out:
1345         if (is_kmalloc)
1346                 kfree(desc_buf);
1347         return ret;
1348 }
1349
1350 /* replace non-printable or non-ASCII characters with spaces */
1351 static inline void ufshcd_remove_non_printable(uint8_t *val)
1352 {
1353         if (!val)
1354                 return;
1355
1356         if (*val < 0x20 || *val > 0x7e)
1357                 *val = ' ';
1358 }
1359
1360 /**
1361  * ufshcd_uic_pwr_ctrl - executes UIC commands (which affects the link power
1362  * state) and waits for it to take effect.
1363  *
1364  */
1365 static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd)
1366 {
1367         unsigned long start = 0;
1368         u8 status;
1369         int ret;
1370
1371         ret = ufshcd_send_uic_cmd(hba, cmd);
1372         if (ret) {
1373                 dev_err(hba->dev,
1374                         "pwr ctrl cmd 0x%x with mode 0x%x uic error %d\n",
1375                         cmd->command, cmd->argument3, ret);
1376
1377                 return ret;
1378         }
1379
1380         start = get_timer(0);
1381         do {
1382                 status = ufshcd_get_upmcrs(hba);
1383                 if (get_timer(start) > UFS_UIC_CMD_TIMEOUT) {
1384                         dev_err(hba->dev,
1385                                 "pwr ctrl cmd 0x%x failed, host upmcrs:0x%x\n",
1386                                 cmd->command, status);
1387                         ret = (status != PWR_OK) ? status : -1;
1388                         break;
1389                 }
1390         } while (status != PWR_LOCAL);
1391
1392         return ret;
1393 }
1394
1395 /**
1396  * ufshcd_uic_change_pwr_mode - Perform the UIC power mode change
1397  *                              using DME_SET primitives.
1398  */
1399 static int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode)
1400 {
1401         struct uic_command uic_cmd = {0};
1402         int ret;
1403
1404         uic_cmd.command = UIC_CMD_DME_SET;
1405         uic_cmd.argument1 = UIC_ARG_MIB(PA_PWRMODE);
1406         uic_cmd.argument3 = mode;
1407         ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
1408
1409         return ret;
1410 }
1411
1412 static
1413 void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufs_hba *hba,
1414                                       struct scsi_cmd *pccb, u32 upiu_flags)
1415 {
1416         struct utp_upiu_req *ucd_req_ptr = hba->ucd_req_ptr;
1417         unsigned int cdb_len;
1418
1419         /* command descriptor fields */
1420         ucd_req_ptr->header.dword_0 =
1421                         UPIU_HEADER_DWORD(UPIU_TRANSACTION_COMMAND, upiu_flags,
1422                                           pccb->lun, TASK_TAG);
1423         ucd_req_ptr->header.dword_1 =
1424                         UPIU_HEADER_DWORD(UPIU_COMMAND_SET_TYPE_SCSI, 0, 0, 0);
1425
1426         /* Total EHS length and Data segment length will be zero */
1427         ucd_req_ptr->header.dword_2 = 0;
1428
1429         ucd_req_ptr->sc.exp_data_transfer_len = cpu_to_be32(pccb->datalen);
1430
1431         cdb_len = min_t(unsigned short, pccb->cmdlen, UFS_CDB_SIZE);
1432         memset(ucd_req_ptr->sc.cdb, 0, UFS_CDB_SIZE);
1433         memcpy(ucd_req_ptr->sc.cdb, pccb->cmd, cdb_len);
1434
1435         memset(hba->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
1436         ufshcd_cache_flush_and_invalidate(ucd_req_ptr, sizeof(*ucd_req_ptr));
1437         ufshcd_cache_flush_and_invalidate(hba->ucd_rsp_ptr, sizeof(*hba->ucd_rsp_ptr));
1438 }
1439
1440 static inline void prepare_prdt_desc(struct ufshcd_sg_entry *entry,
1441                                      unsigned char *buf, ulong len)
1442 {
1443         entry->size = cpu_to_le32(len) | GENMASK(1, 0);
1444         entry->base_addr = cpu_to_le32(lower_32_bits((unsigned long)buf));
1445         entry->upper_addr = cpu_to_le32(upper_32_bits((unsigned long)buf));
1446 }
1447
1448 static void prepare_prdt_table(struct ufs_hba *hba, struct scsi_cmd *pccb)
1449 {
1450         struct utp_transfer_req_desc *req_desc = hba->utrdl;
1451         struct ufshcd_sg_entry *prd_table = hba->ucd_prdt_ptr;
1452         uintptr_t aaddr = (uintptr_t)(pccb->pdata) & ~(ARCH_DMA_MINALIGN - 1);
1453         ulong datalen = pccb->datalen;
1454         int table_length;
1455         u8 *buf;
1456         int i;
1457
1458         if (!datalen) {
1459                 req_desc->prd_table_length = 0;
1460                 ufshcd_cache_flush_and_invalidate(req_desc, sizeof(*req_desc));
1461                 return;
1462         }
1463
1464         if (pccb->dma_dir == DMA_TO_DEVICE) {   /* Write to device */
1465                 flush_dcache_range(aaddr, aaddr +
1466                                    ALIGN(datalen, ARCH_DMA_MINALIGN));
1467         }
1468
1469         /* In any case, invalidate cache to avoid stale data in it. */
1470         invalidate_dcache_range(aaddr, aaddr +
1471                                 ALIGN(datalen, ARCH_DMA_MINALIGN));
1472
1473         table_length = DIV_ROUND_UP(pccb->datalen, MAX_PRDT_ENTRY);
1474         buf = pccb->pdata;
1475         i = table_length;
1476         while (--i) {
1477                 prepare_prdt_desc(&prd_table[table_length - i - 1], buf,
1478                                   MAX_PRDT_ENTRY - 1);
1479                 buf += MAX_PRDT_ENTRY;
1480                 datalen -= MAX_PRDT_ENTRY;
1481         }
1482
1483         prepare_prdt_desc(&prd_table[table_length - i - 1], buf, datalen - 1);
1484
1485         req_desc->prd_table_length = table_length;
1486         ufshcd_cache_flush_and_invalidate(prd_table, sizeof(*prd_table) * table_length);
1487         ufshcd_cache_flush_and_invalidate(req_desc, sizeof(*req_desc));
1488 }
1489
1490 static int ufs_scsi_exec(struct udevice *scsi_dev, struct scsi_cmd *pccb)
1491 {
1492         struct ufs_hba *hba = dev_get_uclass_priv(scsi_dev->parent);
1493         u32 upiu_flags;
1494         int ocs, result = 0;
1495         u8 scsi_status;
1496
1497         ufshcd_prepare_req_desc_hdr(hba, &upiu_flags, pccb->dma_dir);
1498         ufshcd_prepare_utp_scsi_cmd_upiu(hba, pccb, upiu_flags);
1499         prepare_prdt_table(hba, pccb);
1500
1501         ufshcd_send_command(hba, TASK_TAG);
1502
1503         ocs = ufshcd_get_tr_ocs(hba);
1504         switch (ocs) {
1505         case OCS_SUCCESS:
1506                 result = ufshcd_get_req_rsp(hba->ucd_rsp_ptr);
1507                 switch (result) {
1508                 case UPIU_TRANSACTION_RESPONSE:
1509                         result = ufshcd_get_rsp_upiu_result(hba->ucd_rsp_ptr);
1510
1511                         scsi_status = result & MASK_SCSI_STATUS;
1512                         if (scsi_status)
1513                                 return -EINVAL;
1514
1515                         break;
1516                 case UPIU_TRANSACTION_REJECT_UPIU:
1517                         /* TODO: handle Reject UPIU Response */
1518                         dev_err(hba->dev,
1519                                 "Reject UPIU not fully implemented\n");
1520                         return -EINVAL;
1521                 default:
1522                         dev_err(hba->dev,
1523                                 "Unexpected request response code = %x\n",
1524                                 result);
1525                         return -EINVAL;
1526                 }
1527                 break;
1528         default:
1529                 dev_err(hba->dev, "OCS error from controller = %x\n", ocs);
1530                 return -EINVAL;
1531         }
1532
1533         return 0;
1534 }
1535
1536 static inline int ufshcd_read_desc(struct ufs_hba *hba, enum desc_idn desc_id,
1537                                    int desc_index, u8 *buf, u32 size)
1538 {
1539         return ufshcd_read_desc_param(hba, desc_id, desc_index, 0, buf, size);
1540 }
1541
1542 static int ufshcd_read_device_desc(struct ufs_hba *hba, u8 *buf, u32 size)
1543 {
1544         return ufshcd_read_desc(hba, QUERY_DESC_IDN_DEVICE, 0, buf, size);
1545 }
1546
1547 /**
1548  * ufshcd_read_string_desc - read string descriptor
1549  *
1550  */
1551 int ufshcd_read_string_desc(struct ufs_hba *hba, int desc_index,
1552                             u8 *buf, u32 size, bool ascii)
1553 {
1554         int err = 0;
1555
1556         err = ufshcd_read_desc(hba, QUERY_DESC_IDN_STRING, desc_index, buf,
1557                                size);
1558
1559         if (err) {
1560                 dev_err(hba->dev, "%s: reading String Desc failed after %d retries. err = %d\n",
1561                         __func__, QUERY_REQ_RETRIES, err);
1562                 goto out;
1563         }
1564
1565         if (ascii) {
1566                 int desc_len;
1567                 int ascii_len;
1568                 int i;
1569                 u8 *buff_ascii;
1570
1571                 desc_len = buf[0];
1572                 /* remove header and divide by 2 to move from UTF16 to UTF8 */
1573                 ascii_len = (desc_len - QUERY_DESC_HDR_SIZE) / 2 + 1;
1574                 if (size < ascii_len + QUERY_DESC_HDR_SIZE) {
1575                         dev_err(hba->dev, "%s: buffer allocated size is too small\n",
1576                                 __func__);
1577                         err = -ENOMEM;
1578                         goto out;
1579                 }
1580
1581                 buff_ascii = kmalloc(ascii_len, GFP_KERNEL);
1582                 if (!buff_ascii) {
1583                         err = -ENOMEM;
1584                         goto out;
1585                 }
1586
1587                 /*
1588                  * the descriptor contains string in UTF16 format
1589                  * we need to convert to utf-8 so it can be displayed
1590                  */
1591                 utf16_to_utf8(buff_ascii,
1592                               (uint16_t *)&buf[QUERY_DESC_HDR_SIZE], ascii_len);
1593
1594                 /* replace non-printable or non-ASCII characters with spaces */
1595                 for (i = 0; i < ascii_len; i++)
1596                         ufshcd_remove_non_printable(&buff_ascii[i]);
1597
1598                 memset(buf + QUERY_DESC_HDR_SIZE, 0,
1599                        size - QUERY_DESC_HDR_SIZE);
1600                 memcpy(buf + QUERY_DESC_HDR_SIZE, buff_ascii, ascii_len);
1601                 buf[QUERY_DESC_LENGTH_OFFSET] = ascii_len + QUERY_DESC_HDR_SIZE;
1602                 kfree(buff_ascii);
1603         }
1604 out:
1605         return err;
1606 }
1607
1608 static int ufs_get_device_desc(struct ufs_hba *hba,
1609                                struct ufs_dev_desc *dev_desc)
1610 {
1611         int err;
1612         size_t buff_len;
1613         u8 model_index;
1614         u8 *desc_buf;
1615
1616         buff_len = max_t(size_t, hba->desc_size.dev_desc,
1617                          QUERY_DESC_MAX_SIZE + 1);
1618         desc_buf = kmalloc(buff_len, GFP_KERNEL);
1619         if (!desc_buf) {
1620                 err = -ENOMEM;
1621                 goto out;
1622         }
1623
1624         err = ufshcd_read_device_desc(hba, desc_buf, hba->desc_size.dev_desc);
1625         if (err) {
1626                 dev_err(hba->dev, "%s: Failed reading Device Desc. err = %d\n",
1627                         __func__, err);
1628                 goto out;
1629         }
1630
1631         /*
1632          * getting vendor (manufacturerID) and Bank Index in big endian
1633          * format
1634          */
1635         dev_desc->wmanufacturerid = desc_buf[DEVICE_DESC_PARAM_MANF_ID] << 8 |
1636                                      desc_buf[DEVICE_DESC_PARAM_MANF_ID + 1];
1637
1638         model_index = desc_buf[DEVICE_DESC_PARAM_PRDCT_NAME];
1639
1640         /* Zero-pad entire buffer for string termination. */
1641         memset(desc_buf, 0, buff_len);
1642
1643         err = ufshcd_read_string_desc(hba, model_index, desc_buf,
1644                                       QUERY_DESC_MAX_SIZE, true/*ASCII*/);
1645         if (err) {
1646                 dev_err(hba->dev, "%s: Failed reading Product Name. err = %d\n",
1647                         __func__, err);
1648                 goto out;
1649         }
1650
1651         desc_buf[QUERY_DESC_MAX_SIZE] = '\0';
1652         strlcpy(dev_desc->model, (char *)(desc_buf + QUERY_DESC_HDR_SIZE),
1653                 min_t(u8, desc_buf[QUERY_DESC_LENGTH_OFFSET],
1654                       MAX_MODEL_LEN));
1655
1656         /* Null terminate the model string */
1657         dev_desc->model[MAX_MODEL_LEN] = '\0';
1658
1659 out:
1660         kfree(desc_buf);
1661         return err;
1662 }
1663
1664 /**
1665  * ufshcd_get_max_pwr_mode - reads the max power mode negotiated with device
1666  */
1667 static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba)
1668 {
1669         struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info;
1670
1671         if (hba->max_pwr_info.is_valid)
1672                 return 0;
1673
1674         if (hba->quirks & UFSHCD_QUIRK_HIBERN_FASTAUTO) {
1675                 pwr_info->pwr_tx = FASTAUTO_MODE;
1676                 pwr_info->pwr_rx = FASTAUTO_MODE;
1677         } else {
1678                 pwr_info->pwr_tx = FAST_MODE;
1679                 pwr_info->pwr_rx = FAST_MODE;
1680         }
1681         pwr_info->hs_rate = PA_HS_MODE_B;
1682
1683         /* Get the connected lane count */
1684         ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDRXDATALANES),
1685                        &pwr_info->lane_rx);
1686         ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
1687                        &pwr_info->lane_tx);
1688
1689         if (!pwr_info->lane_rx || !pwr_info->lane_tx) {
1690                 dev_err(hba->dev, "%s: invalid connected lanes value. rx=%d, tx=%d\n",
1691                         __func__, pwr_info->lane_rx, pwr_info->lane_tx);
1692                 return -EINVAL;
1693         }
1694
1695         /*
1696          * First, get the maximum gears of HS speed.
1697          * If a zero value, it means there is no HSGEAR capability.
1698          * Then, get the maximum gears of PWM speed.
1699          */
1700         ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), &pwr_info->gear_rx);
1701         if (!pwr_info->gear_rx) {
1702                 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
1703                                &pwr_info->gear_rx);
1704                 if (!pwr_info->gear_rx) {
1705                         dev_err(hba->dev, "%s: invalid max pwm rx gear read = %d\n",
1706                                 __func__, pwr_info->gear_rx);
1707                         return -EINVAL;
1708                 }
1709                 pwr_info->pwr_rx = SLOW_MODE;
1710         }
1711
1712         ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR),
1713                             &pwr_info->gear_tx);
1714         if (!pwr_info->gear_tx) {
1715                 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
1716                                     &pwr_info->gear_tx);
1717                 if (!pwr_info->gear_tx) {
1718                         dev_err(hba->dev, "%s: invalid max pwm tx gear read = %d\n",
1719                                 __func__, pwr_info->gear_tx);
1720                         return -EINVAL;
1721                 }
1722                 pwr_info->pwr_tx = SLOW_MODE;
1723         }
1724
1725         hba->max_pwr_info.is_valid = true;
1726         return 0;
1727 }
1728
1729 static int ufshcd_change_power_mode(struct ufs_hba *hba,
1730                                     struct ufs_pa_layer_attr *pwr_mode)
1731 {
1732         int ret;
1733
1734         /* if already configured to the requested pwr_mode */
1735         if (pwr_mode->gear_rx == hba->pwr_info.gear_rx &&
1736             pwr_mode->gear_tx == hba->pwr_info.gear_tx &&
1737             pwr_mode->lane_rx == hba->pwr_info.lane_rx &&
1738             pwr_mode->lane_tx == hba->pwr_info.lane_tx &&
1739             pwr_mode->pwr_rx == hba->pwr_info.pwr_rx &&
1740             pwr_mode->pwr_tx == hba->pwr_info.pwr_tx &&
1741             pwr_mode->hs_rate == hba->pwr_info.hs_rate) {
1742                 dev_dbg(hba->dev, "%s: power already configured\n", __func__);
1743                 return 0;
1744         }
1745
1746         /*
1747          * Configure attributes for power mode change with below.
1748          * - PA_RXGEAR, PA_ACTIVERXDATALANES, PA_RXTERMINATION,
1749          * - PA_TXGEAR, PA_ACTIVETXDATALANES, PA_TXTERMINATION,
1750          * - PA_HSSERIES
1751          */
1752         ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXGEAR), pwr_mode->gear_rx);
1753         ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVERXDATALANES),
1754                        pwr_mode->lane_rx);
1755         if (pwr_mode->pwr_rx == FASTAUTO_MODE || pwr_mode->pwr_rx == FAST_MODE)
1756                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), TRUE);
1757         else
1758                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), FALSE);
1759
1760         ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXGEAR), pwr_mode->gear_tx);
1761         ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVETXDATALANES),
1762                        pwr_mode->lane_tx);
1763         if (pwr_mode->pwr_tx == FASTAUTO_MODE || pwr_mode->pwr_tx == FAST_MODE)
1764                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), TRUE);
1765         else
1766                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), FALSE);
1767
1768         if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
1769             pwr_mode->pwr_tx == FASTAUTO_MODE ||
1770             pwr_mode->pwr_rx == FAST_MODE ||
1771             pwr_mode->pwr_tx == FAST_MODE)
1772                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HSSERIES),
1773                                pwr_mode->hs_rate);
1774
1775         ret = ufshcd_uic_change_pwr_mode(hba, pwr_mode->pwr_rx << 4 |
1776                                          pwr_mode->pwr_tx);
1777
1778         if (ret) {
1779                 dev_err(hba->dev,
1780                         "%s: power mode change failed %d\n", __func__, ret);
1781
1782                 return ret;
1783         }
1784
1785         /* Copy new Power Mode to power info */
1786         memcpy(&hba->pwr_info, pwr_mode, sizeof(struct ufs_pa_layer_attr));
1787
1788         return ret;
1789 }
1790
1791 /**
1792  * ufshcd_verify_dev_init() - Verify device initialization
1793  *
1794  */
1795 static int ufshcd_verify_dev_init(struct ufs_hba *hba)
1796 {
1797         int retries;
1798         int err;
1799
1800         for (retries = NOP_OUT_RETRIES; retries > 0; retries--) {
1801                 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_NOP,
1802                                           NOP_OUT_TIMEOUT);
1803                 if (!err || err == -ETIMEDOUT)
1804                         break;
1805
1806                 dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err);
1807         }
1808
1809         if (err)
1810                 dev_err(hba->dev, "%s: NOP OUT failed %d\n", __func__, err);
1811
1812         return err;
1813 }
1814
1815 /**
1816  * ufshcd_complete_dev_init() - checks device readiness
1817  */
1818 static int ufshcd_complete_dev_init(struct ufs_hba *hba)
1819 {
1820         int i;
1821         int err;
1822         bool flag_res = 1;
1823
1824         err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
1825                                       QUERY_FLAG_IDN_FDEVICEINIT, NULL);
1826         if (err) {
1827                 dev_err(hba->dev,
1828                         "%s setting fDeviceInit flag failed with error %d\n",
1829                         __func__, err);
1830                 goto out;
1831         }
1832
1833         /* poll for max. 1000 iterations for fDeviceInit flag to clear */
1834         for (i = 0; i < 1000 && !err && flag_res; i++)
1835                 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG,
1836                                               QUERY_FLAG_IDN_FDEVICEINIT,
1837                                               &flag_res);
1838
1839         if (err)
1840                 dev_err(hba->dev,
1841                         "%s reading fDeviceInit flag failed with error %d\n",
1842                         __func__, err);
1843         else if (flag_res)
1844                 dev_err(hba->dev,
1845                         "%s fDeviceInit was not cleared by the device\n",
1846                         __func__);
1847
1848 out:
1849         return err;
1850 }
1851
1852 static void ufshcd_def_desc_sizes(struct ufs_hba *hba)
1853 {
1854         hba->desc_size.dev_desc = QUERY_DESC_DEVICE_DEF_SIZE;
1855         hba->desc_size.pwr_desc = QUERY_DESC_POWER_DEF_SIZE;
1856         hba->desc_size.interc_desc = QUERY_DESC_INTERCONNECT_DEF_SIZE;
1857         hba->desc_size.conf_desc = QUERY_DESC_CONFIGURATION_DEF_SIZE;
1858         hba->desc_size.unit_desc = QUERY_DESC_UNIT_DEF_SIZE;
1859         hba->desc_size.geom_desc = QUERY_DESC_GEOMETRY_DEF_SIZE;
1860         hba->desc_size.hlth_desc = QUERY_DESC_HEALTH_DEF_SIZE;
1861 }
1862
1863 int ufs_start(struct ufs_hba *hba)
1864 {
1865         struct ufs_dev_desc card = {0};
1866         int ret;
1867
1868         ret = ufshcd_link_startup(hba);
1869         if (ret)
1870                 return ret;
1871
1872         ret = ufshcd_verify_dev_init(hba);
1873         if (ret)
1874                 return ret;
1875
1876         ret = ufshcd_complete_dev_init(hba);
1877         if (ret)
1878                 return ret;
1879
1880         /* Init check for device descriptor sizes */
1881         ufshcd_init_desc_sizes(hba);
1882
1883         ret = ufs_get_device_desc(hba, &card);
1884         if (ret) {
1885                 dev_err(hba->dev, "%s: Failed getting device info. err = %d\n",
1886                         __func__, ret);
1887
1888                 return ret;
1889         }
1890
1891         if (ufshcd_get_max_pwr_mode(hba)) {
1892                 dev_err(hba->dev,
1893                         "%s: Failed getting max supported power mode\n",
1894                         __func__);
1895         } else {
1896                 ret = ufshcd_change_power_mode(hba, &hba->max_pwr_info.info);
1897                 if (ret) {
1898                         dev_err(hba->dev, "%s: Failed setting power mode, err = %d\n",
1899                                 __func__, ret);
1900
1901                         return ret;
1902                 }
1903
1904                 printf("Device at %s up at:", hba->dev->name);
1905                 ufshcd_print_pwr_info(hba);
1906         }
1907
1908         return 0;
1909 }
1910
1911 int ufshcd_probe(struct udevice *ufs_dev, struct ufs_hba_ops *hba_ops)
1912 {
1913         struct ufs_hba *hba = dev_get_uclass_priv(ufs_dev);
1914         struct scsi_plat *scsi_plat;
1915         struct udevice *scsi_dev;
1916         void __iomem *mmio_base;
1917         int err;
1918
1919         device_find_first_child(ufs_dev, &scsi_dev);
1920         if (!scsi_dev)
1921                 return -ENODEV;
1922
1923         scsi_plat = dev_get_uclass_plat(scsi_dev);
1924         scsi_plat->max_id = UFSHCD_MAX_ID;
1925         scsi_plat->max_lun = UFS_MAX_LUNS;
1926         scsi_plat->max_bytes_per_req = UFS_MAX_BYTES;
1927
1928         hba->dev = ufs_dev;
1929         hba->ops = hba_ops;
1930
1931         if (device_is_on_pci_bus(ufs_dev)) {
1932                 mmio_base = dm_pci_map_bar(ufs_dev, PCI_BASE_ADDRESS_0, 0, 0,
1933                                            PCI_REGION_TYPE, PCI_REGION_MEM);
1934         } else {
1935                 mmio_base = dev_read_addr_ptr(ufs_dev);
1936         }
1937         hba->mmio_base = mmio_base;
1938
1939         /* Set descriptor lengths to specification defaults */
1940         ufshcd_def_desc_sizes(hba);
1941
1942         ufshcd_ops_init(hba);
1943
1944         /* Read capabilties registers */
1945         hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES);
1946         if (hba->quirks & UFSHCD_QUIRK_BROKEN_64BIT_ADDRESS)
1947                 hba->capabilities &= ~MASK_64_ADDRESSING_SUPPORT;
1948
1949         /* Get UFS version supported by the controller */
1950         hba->version = ufshcd_get_ufs_version(hba);
1951         if (hba->version != UFSHCI_VERSION_10 &&
1952             hba->version != UFSHCI_VERSION_11 &&
1953             hba->version != UFSHCI_VERSION_20 &&
1954             hba->version != UFSHCI_VERSION_21 &&
1955             hba->version != UFSHCI_VERSION_30 &&
1956             hba->version != UFSHCI_VERSION_31)
1957                 dev_err(hba->dev, "invalid UFS version 0x%x\n",
1958                         hba->version);
1959
1960         /* Get Interrupt bit mask per version */
1961         hba->intr_mask = ufshcd_get_intr_mask(hba);
1962
1963         /* Allocate memory for host memory space */
1964         err = ufshcd_memory_alloc(hba);
1965         if (err) {
1966                 dev_err(hba->dev, "Memory allocation failed\n");
1967                 return err;
1968         }
1969
1970         /* Configure Local data structures */
1971         ufshcd_host_memory_configure(hba);
1972
1973         /*
1974          * In order to avoid any spurious interrupt immediately after
1975          * registering UFS controller interrupt handler, clear any pending UFS
1976          * interrupt status and disable all the UFS interrupts.
1977          */
1978         ufshcd_writel(hba, ufshcd_readl(hba, REG_INTERRUPT_STATUS),
1979                       REG_INTERRUPT_STATUS);
1980         ufshcd_writel(hba, 0, REG_INTERRUPT_ENABLE);
1981
1982         err = ufshcd_hba_enable(hba);
1983         if (err) {
1984                 dev_err(hba->dev, "Host controller enable failed\n");
1985                 return err;
1986         }
1987
1988         err = ufs_start(hba);
1989         if (err)
1990                 return err;
1991
1992         return 0;
1993 }
1994
1995 int ufs_scsi_bind(struct udevice *ufs_dev, struct udevice **scsi_devp)
1996 {
1997         int ret = device_bind_driver(ufs_dev, "ufs_scsi", "ufs_scsi",
1998                                      scsi_devp);
1999
2000         return ret;
2001 }
2002
2003 #if IS_ENABLED(CONFIG_BOUNCE_BUFFER)
2004 static int ufs_scsi_buffer_aligned(struct udevice *scsi_dev, struct bounce_buffer *state)
2005 {
2006 #ifdef CONFIG_PHYS_64BIT
2007         struct ufs_hba *hba = dev_get_uclass_priv(scsi_dev->parent);
2008         uintptr_t ubuf = (uintptr_t)state->user_buffer;
2009         size_t len = state->len_aligned;
2010
2011         /* Check if below 32bit boundary */
2012         if ((hba->quirks & UFSHCD_QUIRK_BROKEN_64BIT_ADDRESS) &&
2013             ((ubuf >> 32) || (ubuf + len) >> 32)) {
2014                 dev_dbg(scsi_dev, "Buffer above 32bit boundary %lx-%lx\n",
2015                         ubuf, ubuf + len);
2016                 return 0;
2017         }
2018 #endif
2019         return 1;
2020 }
2021 #endif  /* CONFIG_BOUNCE_BUFFER */
2022
2023 static struct scsi_ops ufs_ops = {
2024         .exec           = ufs_scsi_exec,
2025 #if IS_ENABLED(CONFIG_BOUNCE_BUFFER)
2026         .buffer_aligned = ufs_scsi_buffer_aligned,
2027 #endif  /* CONFIG_BOUNCE_BUFFER */
2028 };
2029
2030 int ufs_probe_dev(int index)
2031 {
2032         struct udevice *dev;
2033
2034         return uclass_get_device(UCLASS_UFS, index, &dev);
2035 }
2036
2037 int ufs_probe(void)
2038 {
2039         struct udevice *dev;
2040         int ret, i;
2041
2042         for (i = 0;; i++) {
2043                 ret = uclass_get_device(UCLASS_UFS, i, &dev);
2044                 if (ret == -ENODEV)
2045                         break;
2046         }
2047
2048         return 0;
2049 }
2050
2051 U_BOOT_DRIVER(ufs_scsi) = {
2052         .id = UCLASS_SCSI,
2053         .name = "ufs_scsi",
2054         .ops = &ufs_ops,
2055 };