drivers/ufs/ufs.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 /**
   3  * ufs.c - Universal Flash Subsystem (UFS) driver
   4  *
   5  * Taken from Linux Kernel v5.2 (drivers/scsi/ufs/ufshcd.c) and ported
   6  * to u-boot.
   7  *
   8  * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com
   9  */
  10
  11 #include <charset.h>
  12 #include <common.h>
  13 #include <dm.h>
  14 #include <log.h>
  15 #include <dm/device_compat.h>
  16 #include <dm/devres.h>
  17 #include <dm/lists.h>
  18 #include <dm/device-internal.h>
  19 #include <malloc.h>
  20 #include <hexdump.h>
  21 #include <scsi.h>
  22 #include <asm/io.h>
  23 #include <asm/dma-mapping.h>
  24 #include <linux/bitops.h>
  25 #include <linux/delay.h>
  26 #include <linux/dma-mapping.h>
  27
  28 #include "ufs.h"
  29
  30 #define UFSHCD_ENABLE_INTRS     (UTP_TRANSFER_REQ_COMPL |\
  31                                  UTP_TASK_REQ_COMPL |\
  32                                  UFSHCD_ERROR_MASK)
  33 /* maximum number of link-startup retries */
  34 #define DME_LINKSTARTUP_RETRIES 3
  35
  36 /* maximum number of retries for a general UIC command  */
  37 #define UFS_UIC_COMMAND_RETRIES 3
  38
  39 /* Query request retries */
  40 #define QUERY_REQ_RETRIES 3
  41 /* Query request timeout */
  42 #define QUERY_REQ_TIMEOUT 1500 /* 1.5 seconds */
  43
  44 /* maximum timeout in ms for a general UIC command */
  45 #define UFS_UIC_CMD_TIMEOUT     1000
  46 /* NOP OUT retries waiting for NOP IN response */
  47 #define NOP_OUT_RETRIES    10
  48 /* Timeout after 30 msecs if NOP OUT hangs without response */
  49 #define NOP_OUT_TIMEOUT    30 /* msecs */
  50
  51 /* Only use one Task Tag for all requests */
  52 #define TASK_TAG        0
  53
  54 /* Expose the flag value from utp_upiu_query.value */
  55 #define MASK_QUERY_UPIU_FLAG_LOC 0xFF
  56
  57 #define MAX_PRDT_ENTRY  262144
  58
  59 /* maximum bytes per request */
  60 #define UFS_MAX_BYTES   (128 * 256 * 1024)
  61
  62 static inline bool ufshcd_is_hba_active(struct ufs_hba *hba);
  63 static inline void ufshcd_hba_stop(struct ufs_hba *hba);
  64 static int ufshcd_hba_enable(struct ufs_hba *hba);
  65
  66 /*
  67  * ufshcd_wait_for_register - wait for register value to change
  68  */
  69 static int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask,
  70                                     u32 val, unsigned long timeout_ms)
  71 {
  72         int err = 0;
  73         unsigned long start = get_timer(0);
  74
  75         /* ignore bits that we don't intend to wait on */
  76         val = val & mask;
  77
  78         while ((ufshcd_readl(hba, reg) & mask) != val) {
  79                 if (get_timer(start) > timeout_ms) {
  80                         if ((ufshcd_readl(hba, reg) & mask) != val)
  81                                 err = -ETIMEDOUT;
  82                         break;
  83                 }
  84         }
  85
  86         return err;
  87 }
  88
  89 /**
  90  * ufshcd_init_pwr_info - setting the POR (power on reset)
  91  * values in hba power info
  92  */
  93 static void ufshcd_init_pwr_info(struct ufs_hba *hba)
  94 {
  95         hba->pwr_info.gear_rx = UFS_PWM_G1;
  96         hba->pwr_info.gear_tx = UFS_PWM_G1;
  97         hba->pwr_info.lane_rx = 1;
  98         hba->pwr_info.lane_tx = 1;
  99         hba->pwr_info.pwr_rx = SLOWAUTO_MODE;
 100         hba->pwr_info.pwr_tx = SLOWAUTO_MODE;
 101         hba->pwr_info.hs_rate = 0;
 102 }
 103
 104 /**
 105  * ufshcd_print_pwr_info - print power params as saved in hba
 106  * power info
 107  */
 108 static void ufshcd_print_pwr_info(struct ufs_hba *hba)
 109 {
 110         static const char * const names[] = {
 111                 "INVALID MODE",
 112                 "FAST MODE",
 113                 "SLOW_MODE",
 114                 "INVALID MODE",
 115                 "FASTAUTO_MODE",
 116                 "SLOWAUTO_MODE",
 117                 "INVALID MODE",
 118         };
 119
 120         dev_err(hba->dev, "[RX, TX]: gear=[%d, %d], lane[%d, %d], pwr[%s, %s], rate = %d\n",
 121                 hba->pwr_info.gear_rx, hba->pwr_info.gear_tx,
 122                 hba->pwr_info.lane_rx, hba->pwr_info.lane_tx,
 123                 names[hba->pwr_info.pwr_rx],
 124                 names[hba->pwr_info.pwr_tx],
 125                 hba->pwr_info.hs_rate);
 126 }
 127
 128 /**
 129  * ufshcd_ready_for_uic_cmd - Check if controller is ready
 130  *                            to accept UIC commands
 131  */
 132 static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba)
 133 {
 134         if (ufshcd_readl(hba, REG_CONTROLLER_STATUS) & UIC_COMMAND_READY)
 135                 return true;
 136         else
 137                 return false;
 138 }
 139
 140 /**
 141  * ufshcd_get_uic_cmd_result - Get the UIC command result
 142  */
 143 static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba)
 144 {
 145         return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2) &
 146                MASK_UIC_COMMAND_RESULT;
 147 }
 148
 149 /**
 150  * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command
 151  */
 152 static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba)
 153 {
 154         return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3);
 155 }
 156
 157 /**
 158  * ufshcd_is_device_present - Check if any device connected to
 159  *                            the host controller
 160  */
 161 static inline bool ufshcd_is_device_present(struct ufs_hba *hba)
 162 {
 163         return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) &
 164                                                 DEVICE_PRESENT) ? true : false;
 165 }
 166
 167 /**
 168  * ufshcd_send_uic_cmd - UFS Interconnect layer command API
 169  *
 170  */
 171 static int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
 172 {
 173         unsigned long start = 0;
 174         u32 intr_status;
 175         u32 enabled_intr_status;
 176
 177         if (!ufshcd_ready_for_uic_cmd(hba)) {
 178                 dev_err(hba->dev,
 179                         "Controller not ready to accept UIC commands\n");
 180                 return -EIO;
 181         }
 182
 183         debug("sending uic command:%d\n", uic_cmd->command);
 184
 185         /* Write Args */
 186         ufshcd_writel(hba, uic_cmd->argument1, REG_UIC_COMMAND_ARG_1);
 187         ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2);
 188         ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3);
 189
 190         /* Write UIC Cmd */
 191         ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK,
 192                       REG_UIC_COMMAND);
 193
 194         start = get_timer(0);
 195         do {
 196                 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
 197                 enabled_intr_status = intr_status & hba->intr_mask;
 198                 ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS);
 199
 200                 if (get_timer(start) > UFS_UIC_CMD_TIMEOUT) {
 201                         dev_err(hba->dev,
 202                                 "Timedout waiting for UIC response\n");
 203
 204                         return -ETIMEDOUT;
 205                 }
 206
 207                 if (enabled_intr_status & UFSHCD_ERROR_MASK) {
 208                         dev_err(hba->dev, "Error in status:%08x\n",
 209                                 enabled_intr_status);
 210
 211                         return -1;
 212                 }
 213         } while (!(enabled_intr_status & UFSHCD_UIC_MASK));
 214
 215         uic_cmd->argument2 = ufshcd_get_uic_cmd_result(hba);
 216         uic_cmd->argument3 = ufshcd_get_dme_attr_val(hba);
 217
 218         debug("Sent successfully\n");
 219
 220         return 0;
 221 }
 222
 223 /**
 224  * ufshcd_dme_set_attr - UIC command for DME_SET, DME_PEER_SET
 225  *
 226  */
 227 int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel, u8 attr_set,
 228                         u32 mib_val, u8 peer)
 229 {
 230         struct uic_command uic_cmd = {0};
 231         static const char *const action[] = {
 232                 "dme-set",
 233                 "dme-peer-set"
 234         };
 235         const char *set = action[!!peer];
 236         int ret;
 237         int retries = UFS_UIC_COMMAND_RETRIES;
 238
 239         uic_cmd.command = peer ?
 240                 UIC_CMD_DME_PEER_SET : UIC_CMD_DME_SET;
 241         uic_cmd.argument1 = attr_sel;
 242         uic_cmd.argument2 = UIC_ARG_ATTR_TYPE(attr_set);
 243         uic_cmd.argument3 = mib_val;
 244
 245         do {
 246                 /* for peer attributes we retry upon failure */
 247                 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
 248                 if (ret)
 249                         dev_dbg(hba->dev, "%s: attr-id 0x%x val 0x%x error code %d\n",
 250                                 set, UIC_GET_ATTR_ID(attr_sel), mib_val, ret);
 251         } while (ret && peer && --retries);
 252
 253         if (ret)
 254                 dev_err(hba->dev, "%s: attr-id 0x%x val 0x%x failed %d retries\n",
 255                         set, UIC_GET_ATTR_ID(attr_sel), mib_val,
 256                         UFS_UIC_COMMAND_RETRIES - retries);
 257
 258         return ret;
 259 }
 260
 261 /**
 262  * ufshcd_dme_get_attr - UIC command for DME_GET, DME_PEER_GET
 263  *
 264  */
 265 int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel,
 266                         u32 *mib_val, u8 peer)
 267 {
 268         struct uic_command uic_cmd = {0};
 269         static const char *const action[] = {
 270                 "dme-get",
 271                 "dme-peer-get"
 272         };
 273         const char *get = action[!!peer];
 274         int ret;
 275         int retries = UFS_UIC_COMMAND_RETRIES;
 276
 277         uic_cmd.command = peer ?
 278                 UIC_CMD_DME_PEER_GET : UIC_CMD_DME_GET;
 279         uic_cmd.argument1 = attr_sel;
 280
 281         do {
 282                 /* for peer attributes we retry upon failure */
 283                 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
 284                 if (ret)
 285                         dev_dbg(hba->dev, "%s: attr-id 0x%x error code %d\n",
 286                                 get, UIC_GET_ATTR_ID(attr_sel), ret);
 287         } while (ret && peer && --retries);
 288
 289         if (ret)
 290                 dev_err(hba->dev, "%s: attr-id 0x%x failed %d retries\n",
 291                         get, UIC_GET_ATTR_ID(attr_sel),
 292                         UFS_UIC_COMMAND_RETRIES - retries);
 293
 294         if (mib_val && !ret)
 295                 *mib_val = uic_cmd.argument3;
 296
 297         return ret;
 298 }
 299
 300 static int ufshcd_disable_tx_lcc(struct ufs_hba *hba, bool peer)
 301 {
 302         u32 tx_lanes, i, err = 0;
 303
 304         if (!peer)
 305                 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
 306                                &tx_lanes);
 307         else
 308                 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
 309                                     &tx_lanes);
 310         for (i = 0; i < tx_lanes; i++) {
 311                 if (!peer)
 312                         err = ufshcd_dme_set(hba,
 313                                              UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
 314                                              UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
 315                                              0);
 316                 else
 317                         err = ufshcd_dme_peer_set(hba,
 318                                         UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
 319                                         UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
 320                                         0);
 321                 if (err) {
 322                         dev_err(hba->dev, "%s: TX LCC Disable failed, peer = %d, lane = %d, err = %d",
 323                                 __func__, peer, i, err);
 324                         break;
 325                 }
 326         }
 327
 328         return err;
 329 }
 330
 331 static inline int ufshcd_disable_device_tx_lcc(struct ufs_hba *hba)
 332 {
 333         return ufshcd_disable_tx_lcc(hba, true);
 334 }
 335
 336 /**
 337  * ufshcd_dme_link_startup - Notify Unipro to perform link startup
 338  *
 339  */
 340 static int ufshcd_dme_link_startup(struct ufs_hba *hba)
 341 {
 342         struct uic_command uic_cmd = {0};
 343         int ret;
 344
 345         uic_cmd.command = UIC_CMD_DME_LINK_STARTUP;
 346
 347         ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
 348         if (ret)
 349                 dev_dbg(hba->dev,
 350                         "dme-link-startup: error code %d\n", ret);
 351         return ret;
 352 }
 353
 354 /**
 355  * ufshcd_disable_intr_aggr - Disables interrupt aggregation.
 356  *
 357  */
 358 static inline void ufshcd_disable_intr_aggr(struct ufs_hba *hba)
 359 {
 360         ufshcd_writel(hba, 0, REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
 361 }
 362
 363 /**
 364  * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY
 365  */
 366 static inline int ufshcd_get_lists_status(u32 reg)
 367 {
 368         return !((reg & UFSHCD_STATUS_READY) == UFSHCD_STATUS_READY);
 369 }
 370
 371 /**
 372  * ufshcd_enable_run_stop_reg - Enable run-stop registers,
 373  *                      When run-stop registers are set to 1, it indicates the
 374  *                      host controller that it can process the requests
 375  */
 376 static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba)
 377 {
 378         ufshcd_writel(hba, UTP_TASK_REQ_LIST_RUN_STOP_BIT,
 379                       REG_UTP_TASK_REQ_LIST_RUN_STOP);
 380         ufshcd_writel(hba, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT,
 381                       REG_UTP_TRANSFER_REQ_LIST_RUN_STOP);
 382 }
 383
 384 /**
 385  * ufshcd_enable_intr - enable interrupts
 386  */
 387 static void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs)
 388 {
 389         u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
 390         u32 rw;
 391
 392         if (hba->version == UFSHCI_VERSION_10) {
 393                 rw = set & INTERRUPT_MASK_RW_VER_10;
 394                 set = rw | ((set ^ intrs) & intrs);
 395         } else {
 396                 set |= intrs;
 397         }
 398
 399         ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
 400
 401         hba->intr_mask = set;
 402 }
 403
 404 /**
 405  * ufshcd_make_hba_operational - Make UFS controller operational
 406  *
 407  * To bring UFS host controller to operational state,
 408  * 1. Enable required interrupts
 409  * 2. Configure interrupt aggregation
 410  * 3. Program UTRL and UTMRL base address
 411  * 4. Configure run-stop-registers
 412  *
 413  */
 414 static int ufshcd_make_hba_operational(struct ufs_hba *hba)
 415 {
 416         int err = 0;
 417         u32 reg;
 418
 419         /* Enable required interrupts */
 420         ufshcd_enable_intr(hba, UFSHCD_ENABLE_INTRS);
 421
 422         /* Disable interrupt aggregation */
 423         ufshcd_disable_intr_aggr(hba);
 424
 425         /* Configure UTRL and UTMRL base address registers */
 426         ufshcd_writel(hba, lower_32_bits((dma_addr_t)hba->utrdl),
 427                       REG_UTP_TRANSFER_REQ_LIST_BASE_L);
 428         ufshcd_writel(hba, upper_32_bits((dma_addr_t)hba->utrdl),
 429                       REG_UTP_TRANSFER_REQ_LIST_BASE_H);
 430         ufshcd_writel(hba, lower_32_bits((dma_addr_t)hba->utmrdl),
 431                       REG_UTP_TASK_REQ_LIST_BASE_L);
 432         ufshcd_writel(hba, upper_32_bits((dma_addr_t)hba->utmrdl),
 433                       REG_UTP_TASK_REQ_LIST_BASE_H);
 434
 435         /*
 436          * UCRDY, UTMRLDY and UTRLRDY bits must be 1
 437          */
 438         reg = ufshcd_readl(hba, REG_CONTROLLER_STATUS);
 439         if (!(ufshcd_get_lists_status(reg))) {
 440                 ufshcd_enable_run_stop_reg(hba);
 441         } else {
 442                 dev_err(hba->dev,
 443                         "Host controller not ready to process requests");
 444                 err = -EIO;
 445                 goto out;
 446         }
 447
 448 out:
 449         return err;
 450 }
 451
 452 /**
 453  * ufshcd_link_startup - Initialize unipro link startup
 454  */
 455 static int ufshcd_link_startup(struct ufs_hba *hba)
 456 {
 457         int ret;
 458         int retries = DME_LINKSTARTUP_RETRIES;
 459         bool link_startup_again = true;
 460
 461 link_startup:
 462         do {
 463                 ufshcd_ops_link_startup_notify(hba, PRE_CHANGE);
 464
 465                 ret = ufshcd_dme_link_startup(hba);
 466
 467                 /* check if device is detected by inter-connect layer */
 468                 if (!ret && !ufshcd_is_device_present(hba)) {
 469                         dev_err(hba->dev, "%s: Device not present\n", __func__);
 470                         ret = -ENXIO;
 471                         goto out;
 472                 }
 473
 474                 /*
 475                  * DME link lost indication is only received when link is up,
 476                  * but we can't be sure if the link is up until link startup
 477                  * succeeds. So reset the local Uni-Pro and try again.
 478                  */
 479                 if (ret && ufshcd_hba_enable(hba))
 480                         goto out;
 481         } while (ret && retries--);
 482
 483         if (ret)
 484                 /* failed to get the link up... retire */
 485                 goto out;
 486
 487         if (link_startup_again) {
 488                 link_startup_again = false;
 489                 retries = DME_LINKSTARTUP_RETRIES;
 490                 goto link_startup;
 491         }
 492
 493         /* Mark that link is up in PWM-G1, 1-lane, SLOW-AUTO mode */
 494         ufshcd_init_pwr_info(hba);
 495
 496         if (hba->quirks & UFSHCD_QUIRK_BROKEN_LCC) {
 497                 ret = ufshcd_disable_device_tx_lcc(hba);
 498                 if (ret)
 499                         goto out;
 500         }
 501
 502         /* Include any host controller configuration via UIC commands */
 503         ret = ufshcd_ops_link_startup_notify(hba, POST_CHANGE);
 504         if (ret)
 505                 goto out;
 506
 507         ret = ufshcd_make_hba_operational(hba);
 508 out:
 509         if (ret)
 510                 dev_err(hba->dev, "link startup failed %d\n", ret);
 511
 512         return ret;
 513 }
 514
 515 /**
 516  * ufshcd_hba_stop - Send controller to reset state
 517  */
 518 static inline void ufshcd_hba_stop(struct ufs_hba *hba)
 519 {
 520         int err;
 521
 522         ufshcd_writel(hba, CONTROLLER_DISABLE,  REG_CONTROLLER_ENABLE);
 523         err = ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE,
 524                                        CONTROLLER_ENABLE, CONTROLLER_DISABLE,
 525                                        10);
 526         if (err)
 527                 dev_err(hba->dev, "%s: Controller disable failed\n", __func__);
 528 }
 529
 530 /**
 531  * ufshcd_is_hba_active - Get controller state
 532  */
 533 static inline bool ufshcd_is_hba_active(struct ufs_hba *hba)
 534 {
 535         return (ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & CONTROLLER_ENABLE)
 536                 ? false : true;
 537 }
 538
 539 /**
 540  * ufshcd_hba_start - Start controller initialization sequence
 541  */
 542 static inline void ufshcd_hba_start(struct ufs_hba *hba)
 543 {
 544         ufshcd_writel(hba, CONTROLLER_ENABLE, REG_CONTROLLER_ENABLE);
 545 }
 546
 547 /**
 548  * ufshcd_hba_enable - initialize the controller
 549  */
 550 static int ufshcd_hba_enable(struct ufs_hba *hba)
 551 {
 552         int retry;
 553
 554         if (!ufshcd_is_hba_active(hba))
 555                 /* change controller state to "reset state" */
 556                 ufshcd_hba_stop(hba);
 557
 558         ufshcd_ops_hce_enable_notify(hba, PRE_CHANGE);
 559
 560         /* start controller initialization sequence */
 561         ufshcd_hba_start(hba);
 562
 563         /*
 564          * To initialize a UFS host controller HCE bit must be set to 1.
 565          * During initialization the HCE bit value changes from 1->0->1.
 566          * When the host controller completes initialization sequence
 567          * it sets the value of HCE bit to 1. The same HCE bit is read back
 568          * to check if the controller has completed initialization sequence.
 569          * So without this delay the value HCE = 1, set in the previous
 570          * instruction might be read back.
 571          * This delay can be changed based on the controller.
 572          */
 573         mdelay(1);
 574
 575         /* wait for the host controller to complete initialization */
 576         retry = 10;
 577         while (ufshcd_is_hba_active(hba)) {
 578                 if (retry) {
 579                         retry--;
 580                 } else {
 581                         dev_err(hba->dev, "Controller enable failed\n");
 582                         return -EIO;
 583                 }
 584                 mdelay(5);
 585         }
 586
 587         /* enable UIC related interrupts */
 588         ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
 589
 590         ufshcd_ops_hce_enable_notify(hba, POST_CHANGE);
 591
 592         return 0;
 593 }
 594
 595 /**
 596  * ufshcd_host_memory_configure - configure local reference block with
 597  *                              memory offsets
 598  */
 599 static void ufshcd_host_memory_configure(struct ufs_hba *hba)
 600 {
 601         struct utp_transfer_req_desc *utrdlp;
 602         dma_addr_t cmd_desc_dma_addr;
 603         u16 response_offset;
 604         u16 prdt_offset;
 605
 606         utrdlp = hba->utrdl;
 607         cmd_desc_dma_addr = (dma_addr_t)hba->ucdl;
 608
 609         utrdlp->command_desc_base_addr_lo =
 610                                 cpu_to_le32(lower_32_bits(cmd_desc_dma_addr));
 611         utrdlp->command_desc_base_addr_hi =
 612                                 cpu_to_le32(upper_32_bits(cmd_desc_dma_addr));
 613
 614         response_offset = offsetof(struct utp_transfer_cmd_desc, response_upiu);
 615         prdt_offset = offsetof(struct utp_transfer_cmd_desc, prd_table);
 616
 617         utrdlp->response_upiu_offset = cpu_to_le16(response_offset >> 2);
 618         utrdlp->prd_table_offset = cpu_to_le16(prdt_offset >> 2);
 619         utrdlp->response_upiu_length = cpu_to_le16(ALIGNED_UPIU_SIZE >> 2);
 620
 621         hba->ucd_req_ptr = (struct utp_upiu_req *)hba->ucdl;
 622         hba->ucd_rsp_ptr =
 623                 (struct utp_upiu_rsp *)&hba->ucdl->response_upiu;
 624         hba->ucd_prdt_ptr =
 625                 (struct ufshcd_sg_entry *)&hba->ucdl->prd_table;
 626 }
 627
 628 /**
 629  * ufshcd_memory_alloc - allocate memory for host memory space data structures
 630  */
 631 static int ufshcd_memory_alloc(struct ufs_hba *hba)
 632 {
 633         /* Allocate one Transfer Request Descriptor
 634          * Should be aligned to 1k boundary.
 635          */
 636         hba->utrdl = memalign(1024, sizeof(struct utp_transfer_req_desc));
 637         if (!hba->utrdl) {
 638                 dev_err(hba->dev, "Transfer Descriptor memory allocation failed\n");
 639                 return -ENOMEM;
 640         }
 641
 642         /* Allocate one Command Descriptor
 643          * Should be aligned to 1k boundary.
 644          */
 645         hba->ucdl = memalign(1024, sizeof(struct utp_transfer_cmd_desc));
 646         if (!hba->ucdl) {
 647                 dev_err(hba->dev, "Command descriptor memory allocation failed\n");
 648                 return -ENOMEM;
 649         }
 650
 651         return 0;
 652 }
 653
 654 /**
 655  * ufshcd_get_intr_mask - Get the interrupt bit mask
 656  */
 657 static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba)
 658 {
 659         u32 intr_mask = 0;
 660
 661         switch (hba->version) {
 662         case UFSHCI_VERSION_10:
 663                 intr_mask = INTERRUPT_MASK_ALL_VER_10;
 664                 break;
 665         case UFSHCI_VERSION_11:
 666         case UFSHCI_VERSION_20:
 667                 intr_mask = INTERRUPT_MASK_ALL_VER_11;
 668                 break;
 669         case UFSHCI_VERSION_21:
 670         default:
 671                 intr_mask = INTERRUPT_MASK_ALL_VER_21;
 672                 break;
 673         }
 674
 675         return intr_mask;
 676 }
 677
 678 /**
 679  * ufshcd_get_ufs_version - Get the UFS version supported by the HBA
 680  */
 681 static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba)
 682 {
 683         return ufshcd_readl(hba, REG_UFS_VERSION);
 684 }
 685
 686 /**
 687  * ufshcd_get_upmcrs - Get the power mode change request status
 688  */
 689 static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba)
 690 {
 691         return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) >> 8) & 0x7;
 692 }
 693
 694 /**
 695  * ufshcd_prepare_req_desc_hdr() - Fills the requests header
 696  * descriptor according to request
 697  */
 698 static void ufshcd_prepare_req_desc_hdr(struct utp_transfer_req_desc *req_desc,
 699                                         u32 *upiu_flags,
 700                                         enum dma_data_direction cmd_dir)
 701 {
 702         u32 data_direction;
 703         u32 dword_0;
 704
 705         if (cmd_dir == DMA_FROM_DEVICE) {
 706                 data_direction = UTP_DEVICE_TO_HOST;
 707                 *upiu_flags = UPIU_CMD_FLAGS_READ;
 708         } else if (cmd_dir == DMA_TO_DEVICE) {
 709                 data_direction = UTP_HOST_TO_DEVICE;
 710                 *upiu_flags = UPIU_CMD_FLAGS_WRITE;
 711         } else {
 712                 data_direction = UTP_NO_DATA_TRANSFER;
 713                 *upiu_flags = UPIU_CMD_FLAGS_NONE;
 714         }
 715
 716         dword_0 = data_direction | (0x1 << UPIU_COMMAND_TYPE_OFFSET);
 717
 718         /* Enable Interrupt for command */
 719         dword_0 |= UTP_REQ_DESC_INT_CMD;
 720
 721         /* Transfer request descriptor header fields */
 722         req_desc->header.dword_0 = cpu_to_le32(dword_0);
 723         /* dword_1 is reserved, hence it is set to 0 */
 724         req_desc->header.dword_1 = 0;
 725         /*
 726          * assigning invalid value for command status. Controller
 727          * updates OCS on command completion, with the command
 728          * status
 729          */
 730         req_desc->header.dword_2 =
 731                 cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
 732         /* dword_3 is reserved, hence it is set to 0 */
 733         req_desc->header.dword_3 = 0;
 734
 735         req_desc->prd_table_length = 0;
 736 }
 737
 738 static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba,
 739                                               u32 upiu_flags)
 740 {
 741         struct utp_upiu_req *ucd_req_ptr = hba->ucd_req_ptr;
 742         struct ufs_query *query = &hba->dev_cmd.query;
 743         u16 len = be16_to_cpu(query->request.upiu_req.length);
 744
 745         /* Query request header */
 746         ucd_req_ptr->header.dword_0 =
 747                                 UPIU_HEADER_DWORD(UPIU_TRANSACTION_QUERY_REQ,
 748                                                   upiu_flags, 0, TASK_TAG);
 749         ucd_req_ptr->header.dword_1 =
 750                                 UPIU_HEADER_DWORD(0, query->request.query_func,
 751                                                   0, 0);
 752
 753         /* Data segment length only need for WRITE_DESC */
 754         if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
 755                 ucd_req_ptr->header.dword_2 =
 756                                 UPIU_HEADER_DWORD(0, 0, (len >> 8), (u8)len);
 757         else
 758                 ucd_req_ptr->header.dword_2 = 0;
 759
 760         /* Copy the Query Request buffer as is */
 761         memcpy(&ucd_req_ptr->qr, &query->request.upiu_req, QUERY_OSF_SIZE);
 762
 763         /* Copy the Descriptor */
 764         if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
 765                 memcpy(ucd_req_ptr + 1, query->descriptor, len);
 766
 767         memset(hba->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
 768 }
 769
 770 static inline void ufshcd_prepare_utp_nop_upiu(struct ufs_hba *hba)
 771 {
 772         struct utp_upiu_req *ucd_req_ptr = hba->ucd_req_ptr;
 773
 774         memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req));
 775
 776         /* command descriptor fields */
 777         ucd_req_ptr->header.dword_0 =
 778                         UPIU_HEADER_DWORD(UPIU_TRANSACTION_NOP_OUT, 0, 0, 0x1f);
 779         /* clear rest of the fields of basic header */
 780         ucd_req_ptr->header.dword_1 = 0;
 781         ucd_req_ptr->header.dword_2 = 0;
 782
 783         memset(hba->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
 784 }
 785
 786 /**
 787  * ufshcd_comp_devman_upiu - UFS Protocol Information Unit(UPIU)
 788  *                           for Device Management Purposes
 789  */
 790 static int ufshcd_comp_devman_upiu(struct ufs_hba *hba,
 791                                    enum dev_cmd_type cmd_type)
 792 {
 793         u32 upiu_flags;
 794         int ret = 0;
 795         struct utp_transfer_req_desc *req_desc = hba->utrdl;
 796
 797         hba->dev_cmd.type = cmd_type;
 798
 799         ufshcd_prepare_req_desc_hdr(req_desc, &upiu_flags, DMA_NONE);
 800         switch (cmd_type) {
 801         case DEV_CMD_TYPE_QUERY:
 802                 ufshcd_prepare_utp_query_req_upiu(hba, upiu_flags);
 803                 break;
 804         case DEV_CMD_TYPE_NOP:
 805                 ufshcd_prepare_utp_nop_upiu(hba);
 806                 break;
 807         default:
 808                 ret = -EINVAL;
 809         }
 810
 811         return ret;
 812 }
 813
 814 static int ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag)
 815 {
 816         unsigned long start;
 817         u32 intr_status;
 818         u32 enabled_intr_status;
 819
 820         ufshcd_writel(hba, 1 << task_tag, REG_UTP_TRANSFER_REQ_DOOR_BELL);
 821
 822         start = get_timer(0);
 823         do {
 824                 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
 825                 enabled_intr_status = intr_status & hba->intr_mask;
 826                 ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS);
 827
 828                 if (get_timer(start) > QUERY_REQ_TIMEOUT) {
 829                         dev_err(hba->dev,
 830                                 "Timedout waiting for UTP response\n");
 831
 832                         return -ETIMEDOUT;
 833                 }
 834
 835                 if (enabled_intr_status & UFSHCD_ERROR_MASK) {
 836                         dev_err(hba->dev, "Error in status:%08x\n",
 837                                 enabled_intr_status);
 838
 839                         return -1;
 840                 }
 841         } while (!(enabled_intr_status & UTP_TRANSFER_REQ_COMPL));
 842
 843         return 0;
 844 }
 845
 846 /**
 847  * ufshcd_get_req_rsp - returns the TR response transaction type
 848  */
 849 static inline int ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr)
 850 {
 851         return be32_to_cpu(ucd_rsp_ptr->header.dword_0) >> 24;
 852 }
 853
 854 /**
 855  * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status
 856  *
 857  */
 858 static inline int ufshcd_get_tr_ocs(struct ufs_hba *hba)
 859 {
 860         return le32_to_cpu(hba->utrdl->header.dword_2) & MASK_OCS;
 861 }
 862
 863 static inline int ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp *ucd_rsp_ptr)
 864 {
 865         return be32_to_cpu(ucd_rsp_ptr->header.dword_1) & MASK_RSP_UPIU_RESULT;
 866 }
 867
 868 static int ufshcd_check_query_response(struct ufs_hba *hba)
 869 {
 870         struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
 871
 872         /* Get the UPIU response */
 873         query_res->response = ufshcd_get_rsp_upiu_result(hba->ucd_rsp_ptr) >>
 874                                 UPIU_RSP_CODE_OFFSET;
 875         return query_res->response;
 876 }
 877
 878 /**
 879  * ufshcd_copy_query_response() - Copy the Query Response and the data
 880  * descriptor
 881  */
 882 static int ufshcd_copy_query_response(struct ufs_hba *hba)
 883 {
 884         struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
 885
 886         memcpy(&query_res->upiu_res, &hba->ucd_rsp_ptr->qr, QUERY_OSF_SIZE);
 887
 888         /* Get the descriptor */
 889         if (hba->dev_cmd.query.descriptor &&
 890             hba->ucd_rsp_ptr->qr.opcode == UPIU_QUERY_OPCODE_READ_DESC) {
 891                 u8 *descp = (u8 *)hba->ucd_rsp_ptr +
 892                                 GENERAL_UPIU_REQUEST_SIZE;
 893                 u16 resp_len;
 894                 u16 buf_len;
 895
 896                 /* data segment length */
 897                 resp_len = be32_to_cpu(hba->ucd_rsp_ptr->header.dword_2) &
 898                                                 MASK_QUERY_DATA_SEG_LEN;
 899                 buf_len =
 900                         be16_to_cpu(hba->dev_cmd.query.request.upiu_req.length);
 901                 if (likely(buf_len >= resp_len)) {
 902                         memcpy(hba->dev_cmd.query.descriptor, descp, resp_len);
 903                 } else {
 904                         dev_warn(hba->dev,
 905                                  "%s: Response size is bigger than buffer",
 906                                  __func__);
 907                         return -EINVAL;
 908                 }
 909         }
 910
 911         return 0;
 912 }
 913
 914 /**
 915  * ufshcd_exec_dev_cmd - API for sending device management requests
 916  */
 917 static int ufshcd_exec_dev_cmd(struct ufs_hba *hba, enum dev_cmd_type cmd_type,
 918                                int timeout)
 919 {
 920         int err;
 921         int resp;
 922
 923         err = ufshcd_comp_devman_upiu(hba, cmd_type);
 924         if (err)
 925                 return err;
 926
 927         err = ufshcd_send_command(hba, TASK_TAG);
 928         if (err)
 929                 return err;
 930
 931         err = ufshcd_get_tr_ocs(hba);
 932         if (err) {
 933                 dev_err(hba->dev, "Error in OCS:%d\n", err);
 934                 return -EINVAL;
 935         }
 936
 937         resp = ufshcd_get_req_rsp(hba->ucd_rsp_ptr);
 938         switch (resp) {
 939         case UPIU_TRANSACTION_NOP_IN:
 940                 break;
 941         case UPIU_TRANSACTION_QUERY_RSP:
 942                 err = ufshcd_check_query_response(hba);
 943                 if (!err)
 944                         err = ufshcd_copy_query_response(hba);
 945                 break;
 946         case UPIU_TRANSACTION_REJECT_UPIU:
 947                 /* TODO: handle Reject UPIU Response */
 948                 err = -EPERM;
 949                 dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n",
 950                         __func__);
 951                 break;
 952         default:
 953                 err = -EINVAL;
 954                 dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n",
 955                         __func__, resp);
 956         }
 957
 958         return err;
 959 }
 960
 961 /**
 962  * ufshcd_init_query() - init the query response and request parameters
 963  */
 964 static inline void ufshcd_init_query(struct ufs_hba *hba,
 965                                      struct ufs_query_req **request,
 966                                      struct ufs_query_res **response,
 967                                      enum query_opcode opcode,
 968                                      u8 idn, u8 index, u8 selector)
 969 {
 970         *request = &hba->dev_cmd.query.request;
 971         *response = &hba->dev_cmd.query.response;
 972         memset(*request, 0, sizeof(struct ufs_query_req));
 973         memset(*response, 0, sizeof(struct ufs_query_res));
 974         (*request)->upiu_req.opcode = opcode;
 975         (*request)->upiu_req.idn = idn;
 976         (*request)->upiu_req.index = index;
 977         (*request)->upiu_req.selector = selector;
 978 }
 979
 980 /**
 981  * ufshcd_query_flag() - API function for sending flag query requests
 982  */
 983 int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode,
 984                       enum flag_idn idn, bool *flag_res)
 985 {
 986         struct ufs_query_req *request = NULL;
 987         struct ufs_query_res *response = NULL;
 988         int err, index = 0, selector = 0;
 989         int timeout = QUERY_REQ_TIMEOUT;
 990
 991         ufshcd_init_query(hba, &request, &response, opcode, idn, index,
 992                           selector);
 993
 994         switch (opcode) {
 995         case UPIU_QUERY_OPCODE_SET_FLAG:
 996         case UPIU_QUERY_OPCODE_CLEAR_FLAG:
 997         case UPIU_QUERY_OPCODE_TOGGLE_FLAG:
 998                 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
 999                 break;
1000         case UPIU_QUERY_OPCODE_READ_FLAG:
1001                 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
1002                 if (!flag_res) {
1003                         /* No dummy reads */
1004                         dev_err(hba->dev, "%s: Invalid argument for read request\n",
1005                                 __func__);
1006                         err = -EINVAL;
1007                         goto out;
1008                 }
1009                 break;
1010         default:
1011                 dev_err(hba->dev,
1012                         "%s: Expected query flag opcode but got = %d\n",
1013                         __func__, opcode);
1014                 err = -EINVAL;
1015                 goto out;
1016         }
1017
1018         err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, timeout);
1019
1020         if (err) {
1021                 dev_err(hba->dev,
1022                         "%s: Sending flag query for idn %d failed, err = %d\n",
1023                         __func__, idn, err);
1024                 goto out;
1025         }
1026
1027         if (flag_res)
1028                 *flag_res = (be32_to_cpu(response->upiu_res.value) &
1029                                 MASK_QUERY_UPIU_FLAG_LOC) & 0x1;
1030
1031 out:
1032         return err;
1033 }
1034
1035 static int ufshcd_query_flag_retry(struct ufs_hba *hba,
1036                                    enum query_opcode opcode,
1037                                    enum flag_idn idn, bool *flag_res)
1038 {
1039         int ret;
1040         int retries;
1041
1042         for (retries = 0; retries < QUERY_REQ_RETRIES; retries++) {
1043                 ret = ufshcd_query_flag(hba, opcode, idn, flag_res);
1044                 if (ret)
1045                         dev_dbg(hba->dev,
1046                                 "%s: failed with error %d, retries %d\n",
1047                                 __func__, ret, retries);
1048                 else
1049                         break;
1050         }
1051
1052         if (ret)
1053                 dev_err(hba->dev,
1054                         "%s: query attribute, opcode %d, idn %d, failed with error %d after %d retires\n",
1055                         __func__, opcode, idn, ret, retries);
1056         return ret;
1057 }
1058
1059 static int __ufshcd_query_descriptor(struct ufs_hba *hba,
1060                                      enum query_opcode opcode,
1061                                      enum desc_idn idn, u8 index, u8 selector,
1062                                      u8 *desc_buf, int *buf_len)
1063 {
1064         struct ufs_query_req *request = NULL;
1065         struct ufs_query_res *response = NULL;
1066         int err;
1067
1068         if (!desc_buf) {
1069                 dev_err(hba->dev, "%s: descriptor buffer required for opcode 0x%x\n",
1070                         __func__, opcode);
1071                 err = -EINVAL;
1072                 goto out;
1073         }
1074
1075         if (*buf_len < QUERY_DESC_MIN_SIZE || *buf_len > QUERY_DESC_MAX_SIZE) {
1076                 dev_err(hba->dev, "%s: descriptor buffer size (%d) is out of range\n",
1077                         __func__, *buf_len);
1078                 err = -EINVAL;
1079                 goto out;
1080         }
1081
1082         ufshcd_init_query(hba, &request, &response, opcode, idn, index,
1083                           selector);
1084         hba->dev_cmd.query.descriptor = desc_buf;
1085         request->upiu_req.length = cpu_to_be16(*buf_len);
1086
1087         switch (opcode) {
1088         case UPIU_QUERY_OPCODE_WRITE_DESC:
1089                 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
1090                 break;
1091         case UPIU_QUERY_OPCODE_READ_DESC:
1092                 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
1093                 break;
1094         default:
1095                 dev_err(hba->dev, "%s: Expected query descriptor opcode but got = 0x%.2x\n",
1096                         __func__, opcode);
1097                 err = -EINVAL;
1098                 goto out;
1099         }
1100
1101         err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
1102
1103         if (err) {
1104                 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
1105                         __func__, opcode, idn, index, err);
1106                 goto out;
1107         }
1108
1109         hba->dev_cmd.query.descriptor = NULL;
1110         *buf_len = be16_to_cpu(response->upiu_res.length);
1111
1112 out:
1113         return err;
1114 }
1115
1116 /**
1117  * ufshcd_query_descriptor_retry - API function for sending descriptor requests
1118  */
1119 int ufshcd_query_descriptor_retry(struct ufs_hba *hba, enum query_opcode opcode,
1120                                   enum desc_idn idn, u8 index, u8 selector,
1121                                   u8 *desc_buf, int *buf_len)
1122 {
1123         int err;
1124         int retries;
1125
1126         for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
1127                 err = __ufshcd_query_descriptor(hba, opcode, idn, index,
1128                                                 selector, desc_buf, buf_len);
1129                 if (!err || err == -EINVAL)
1130                         break;
1131         }
1132
1133         return err;
1134 }
1135
1136 /**
1137  * ufshcd_read_desc_length - read the specified descriptor length from header
1138  */
1139 static int ufshcd_read_desc_length(struct ufs_hba *hba, enum desc_idn desc_id,
1140                                    int desc_index, int *desc_length)
1141 {
1142         int ret;
1143         u8 header[QUERY_DESC_HDR_SIZE];
1144         int header_len = QUERY_DESC_HDR_SIZE;
1145
1146         if (desc_id >= QUERY_DESC_IDN_MAX)
1147                 return -EINVAL;
1148
1149         ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC,
1150                                             desc_id, desc_index, 0, header,
1151                                             &header_len);
1152
1153         if (ret) {
1154                 dev_err(hba->dev, "%s: Failed to get descriptor header id %d",
1155                         __func__, desc_id);
1156                 return ret;
1157         } else if (desc_id != header[QUERY_DESC_DESC_TYPE_OFFSET]) {
1158                 dev_warn(hba->dev, "%s: descriptor header id %d and desc_id %d mismatch",
1159                          __func__, header[QUERY_DESC_DESC_TYPE_OFFSET],
1160                          desc_id);
1161                 ret = -EINVAL;
1162         }
1163
1164         *desc_length = header[QUERY_DESC_LENGTH_OFFSET];
1165
1166         return ret;
1167 }
1168
1169 static void ufshcd_init_desc_sizes(struct ufs_hba *hba)
1170 {
1171         int err;
1172
1173         err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_DEVICE, 0,
1174                                       &hba->desc_size.dev_desc);
1175         if (err)
1176                 hba->desc_size.dev_desc = QUERY_DESC_DEVICE_DEF_SIZE;
1177
1178         err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_POWER, 0,
1179                                       &hba->desc_size.pwr_desc);
1180         if (err)
1181                 hba->desc_size.pwr_desc = QUERY_DESC_POWER_DEF_SIZE;
1182
1183         err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_INTERCONNECT, 0,
1184                                       &hba->desc_size.interc_desc);
1185         if (err)
1186                 hba->desc_size.interc_desc = QUERY_DESC_INTERCONNECT_DEF_SIZE;
1187
1188         err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_CONFIGURATION, 0,
1189                                       &hba->desc_size.conf_desc);
1190         if (err)
1191                 hba->desc_size.conf_desc = QUERY_DESC_CONFIGURATION_DEF_SIZE;
1192
1193         err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_UNIT, 0,
1194                                       &hba->desc_size.unit_desc);
1195         if (err)
1196                 hba->desc_size.unit_desc = QUERY_DESC_UNIT_DEF_SIZE;
1197
1198         err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_GEOMETRY, 0,
1199                                       &hba->desc_size.geom_desc);
1200         if (err)
1201                 hba->desc_size.geom_desc = QUERY_DESC_GEOMETRY_DEF_SIZE;
1202
1203         err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_HEALTH, 0,
1204                                       &hba->desc_size.hlth_desc);
1205         if (err)
1206                 hba->desc_size.hlth_desc = QUERY_DESC_HEALTH_DEF_SIZE;
1207 }
1208
1209 /**
1210  * ufshcd_map_desc_id_to_length - map descriptor IDN to its length
1211  *
1212  */
1213 int ufshcd_map_desc_id_to_length(struct ufs_hba *hba, enum desc_idn desc_id,
1214                                  int *desc_len)
1215 {
1216         switch (desc_id) {
1217         case QUERY_DESC_IDN_DEVICE:
1218                 *desc_len = hba->desc_size.dev_desc;
1219                 break;
1220         case QUERY_DESC_IDN_POWER:
1221                 *desc_len = hba->desc_size.pwr_desc;
1222                 break;
1223         case QUERY_DESC_IDN_GEOMETRY:
1224                 *desc_len = hba->desc_size.geom_desc;
1225                 break;
1226         case QUERY_DESC_IDN_CONFIGURATION:
1227                 *desc_len = hba->desc_size.conf_desc;
1228                 break;
1229         case QUERY_DESC_IDN_UNIT:
1230                 *desc_len = hba->desc_size.unit_desc;
1231                 break;
1232         case QUERY_DESC_IDN_INTERCONNECT:
1233                 *desc_len = hba->desc_size.interc_desc;
1234                 break;
1235         case QUERY_DESC_IDN_STRING:
1236                 *desc_len = QUERY_DESC_MAX_SIZE;
1237                 break;
1238         case QUERY_DESC_IDN_HEALTH:
1239                 *desc_len = hba->desc_size.hlth_desc;
1240                 break;
1241         case QUERY_DESC_IDN_RFU_0:
1242         case QUERY_DESC_IDN_RFU_1:
1243                 *desc_len = 0;
1244                 break;
1245         default:
1246                 *desc_len = 0;
1247                 return -EINVAL;
1248         }
1249         return 0;
1250 }
1251 EXPORT_SYMBOL(ufshcd_map_desc_id_to_length);
1252
1253 /**
1254  * ufshcd_read_desc_param - read the specified descriptor parameter
1255  *
1256  */
1257 int ufshcd_read_desc_param(struct ufs_hba *hba, enum desc_idn desc_id,
1258                            int desc_index, u8 param_offset, u8 *param_read_buf,
1259                            u8 param_size)
1260 {
1261         int ret;
1262         u8 *desc_buf;
1263         int buff_len;
1264         bool is_kmalloc = true;
1265
1266         /* Safety check */
1267         if (desc_id >= QUERY_DESC_IDN_MAX || !param_size)
1268                 return -EINVAL;
1269
1270         /* Get the max length of descriptor from structure filled up at probe
1271          * time.
1272          */
1273         ret = ufshcd_map_desc_id_to_length(hba, desc_id, &buff_len);
1274
1275         /* Sanity checks */
1276         if (ret || !buff_len) {
1277                 dev_err(hba->dev, "%s: Failed to get full descriptor length",
1278                         __func__);
1279                 return ret;
1280         }
1281
1282         /* Check whether we need temp memory */
1283         if (param_offset != 0 || param_size < buff_len) {
1284                 desc_buf = kmalloc(buff_len, GFP_KERNEL);
1285                 if (!desc_buf)
1286                         return -ENOMEM;
1287         } else {
1288                 desc_buf = param_read_buf;
1289                 is_kmalloc = false;
1290         }
1291
1292         /* Request for full descriptor */
1293         ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC,
1294                                             desc_id, desc_index, 0, desc_buf,
1295                                             &buff_len);
1296
1297         if (ret) {
1298                 dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d, desc_index %d, param_offset %d, ret %d",
1299                         __func__, desc_id, desc_index, param_offset, ret);
1300                 goto out;
1301         }
1302
1303         /* Sanity check */
1304         if (desc_buf[QUERY_DESC_DESC_TYPE_OFFSET] != desc_id) {
1305                 dev_err(hba->dev, "%s: invalid desc_id %d in descriptor header",
1306                         __func__, desc_buf[QUERY_DESC_DESC_TYPE_OFFSET]);
1307                 ret = -EINVAL;
1308                 goto out;
1309         }
1310
1311         /* Check wherher we will not copy more data, than available */
1312         if (is_kmalloc && param_size > buff_len)
1313                 param_size = buff_len;
1314
1315         if (is_kmalloc)
1316                 memcpy(param_read_buf, &desc_buf[param_offset], param_size);
1317 out:
1318         if (is_kmalloc)
1319                 kfree(desc_buf);
1320         return ret;
1321 }
1322
1323 /* replace non-printable or non-ASCII characters with spaces */
1324 static inline void ufshcd_remove_non_printable(uint8_t *val)
1325 {
1326         if (!val)
1327                 return;
1328
1329         if (*val < 0x20 || *val > 0x7e)
1330                 *val = ' ';
1331 }
1332
1333 /**
1334  * ufshcd_uic_pwr_ctrl - executes UIC commands (which affects the link power
1335  * state) and waits for it to take effect.
1336  *
1337  */
1338 static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd)
1339 {
1340         unsigned long start = 0;
1341         u8 status;
1342         int ret;
1343
1344         ret = ufshcd_send_uic_cmd(hba, cmd);
1345         if (ret) {
1346                 dev_err(hba->dev,
1347                         "pwr ctrl cmd 0x%x with mode 0x%x uic error %d\n",
1348                         cmd->command, cmd->argument3, ret);
1349
1350                 return ret;
1351         }
1352
1353         start = get_timer(0);
1354         do {
1355                 status = ufshcd_get_upmcrs(hba);
1356                 if (get_timer(start) > UFS_UIC_CMD_TIMEOUT) {
1357                         dev_err(hba->dev,
1358                                 "pwr ctrl cmd 0x%x failed, host upmcrs:0x%x\n",
1359                                 cmd->command, status);
1360                         ret = (status != PWR_OK) ? status : -1;
1361                         break;
1362                 }
1363         } while (status != PWR_LOCAL);
1364
1365         return ret;
1366 }
1367
1368 /**
1369  * ufshcd_uic_change_pwr_mode - Perform the UIC power mode change
1370  *                              using DME_SET primitives.
1371  */
1372 static int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode)
1373 {
1374         struct uic_command uic_cmd = {0};
1375         int ret;
1376
1377         uic_cmd.command = UIC_CMD_DME_SET;
1378         uic_cmd.argument1 = UIC_ARG_MIB(PA_PWRMODE);
1379         uic_cmd.argument3 = mode;
1380         ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
1381
1382         return ret;
1383 }
1384
1385 static
1386 void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufs_hba *hba,
1387                                       struct scsi_cmd *pccb, u32 upiu_flags)
1388 {
1389         struct utp_upiu_req *ucd_req_ptr = hba->ucd_req_ptr;
1390         unsigned int cdb_len;
1391
1392         /* command descriptor fields */
1393         ucd_req_ptr->header.dword_0 =
1394                         UPIU_HEADER_DWORD(UPIU_TRANSACTION_COMMAND, upiu_flags,
1395                                           pccb->lun, TASK_TAG);
1396         ucd_req_ptr->header.dword_1 =
1397                         UPIU_HEADER_DWORD(UPIU_COMMAND_SET_TYPE_SCSI, 0, 0, 0);
1398
1399         /* Total EHS length and Data segment length will be zero */
1400         ucd_req_ptr->header.dword_2 = 0;
1401
1402         ucd_req_ptr->sc.exp_data_transfer_len = cpu_to_be32(pccb->datalen);
1403
1404         cdb_len = min_t(unsigned short, pccb->cmdlen, UFS_CDB_SIZE);
1405         memset(ucd_req_ptr->sc.cdb, 0, UFS_CDB_SIZE);
1406         memcpy(ucd_req_ptr->sc.cdb, pccb->cmd, cdb_len);
1407
1408         memset(hba->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
1409 }
1410
1411 static inline void prepare_prdt_desc(struct ufshcd_sg_entry *entry,
1412                                      unsigned char *buf, ulong len)
1413 {
1414         entry->size = cpu_to_le32(len) | GENMASK(1, 0);
1415         entry->base_addr = cpu_to_le32(lower_32_bits((unsigned long)buf));
1416         entry->upper_addr = cpu_to_le32(upper_32_bits((unsigned long)buf));
1417 }
1418
1419 static void prepare_prdt_table(struct ufs_hba *hba, struct scsi_cmd *pccb)
1420 {
1421         struct utp_transfer_req_desc *req_desc = hba->utrdl;
1422         struct ufshcd_sg_entry *prd_table = hba->ucd_prdt_ptr;
1423         ulong datalen = pccb->datalen;
1424         int table_length;
1425         u8 *buf;
1426         int i;
1427
1428         if (!datalen) {
1429                 req_desc->prd_table_length = 0;
1430                 return;
1431         }
1432
1433         table_length = DIV_ROUND_UP(pccb->datalen, MAX_PRDT_ENTRY);
1434         buf = pccb->pdata;
1435         i = table_length;
1436         while (--i) {
1437                 prepare_prdt_desc(&prd_table[table_length - i - 1], buf,
1438                                   MAX_PRDT_ENTRY - 1);
1439                 buf += MAX_PRDT_ENTRY;
1440                 datalen -= MAX_PRDT_ENTRY;
1441         }
1442
1443         prepare_prdt_desc(&prd_table[table_length - i - 1], buf, datalen - 1);
1444
1445         req_desc->prd_table_length = table_length;
1446 }
1447
1448 static int ufs_scsi_exec(struct udevice *scsi_dev, struct scsi_cmd *pccb)
1449 {
1450         struct ufs_hba *hba = dev_get_uclass_priv(scsi_dev->parent);
1451         struct utp_transfer_req_desc *req_desc = hba->utrdl;
1452         u32 upiu_flags;
1453         int ocs, result = 0;
1454         u8 scsi_status;
1455
1456         ufshcd_prepare_req_desc_hdr(req_desc, &upiu_flags, pccb->dma_dir);
1457         ufshcd_prepare_utp_scsi_cmd_upiu(hba, pccb, upiu_flags);
1458         prepare_prdt_table(hba, pccb);
1459
1460         ufshcd_send_command(hba, TASK_TAG);
1461
1462         ocs = ufshcd_get_tr_ocs(hba);
1463         switch (ocs) {
1464         case OCS_SUCCESS:
1465                 result = ufshcd_get_req_rsp(hba->ucd_rsp_ptr);
1466                 switch (result) {
1467                 case UPIU_TRANSACTION_RESPONSE:
1468                         result = ufshcd_get_rsp_upiu_result(hba->ucd_rsp_ptr);
1469
1470                         scsi_status = result & MASK_SCSI_STATUS;
1471                         if (scsi_status)
1472                                 return -EINVAL;
1473
1474                         break;
1475                 case UPIU_TRANSACTION_REJECT_UPIU:
1476                         /* TODO: handle Reject UPIU Response */
1477                         dev_err(hba->dev,
1478                                 "Reject UPIU not fully implemented\n");
1479                         return -EINVAL;
1480                 default:
1481                         dev_err(hba->dev,
1482                                 "Unexpected request response code = %x\n",
1483                                 result);
1484                         return -EINVAL;
1485                 }
1486                 break;
1487         default:
1488                 dev_err(hba->dev, "OCS error from controller = %x\n", ocs);
1489                 return -EINVAL;
1490         }
1491
1492         return 0;
1493 }
1494
1495 static inline int ufshcd_read_desc(struct ufs_hba *hba, enum desc_idn desc_id,
1496                                    int desc_index, u8 *buf, u32 size)
1497 {
1498         return ufshcd_read_desc_param(hba, desc_id, desc_index, 0, buf, size);
1499 }
1500
1501 static int ufshcd_read_device_desc(struct ufs_hba *hba, u8 *buf, u32 size)
1502 {
1503         return ufshcd_read_desc(hba, QUERY_DESC_IDN_DEVICE, 0, buf, size);
1504 }
1505
1506 /**
1507  * ufshcd_read_string_desc - read string descriptor
1508  *
1509  */
1510 int ufshcd_read_string_desc(struct ufs_hba *hba, int desc_index,
1511                             u8 *buf, u32 size, bool ascii)
1512 {
1513         int err = 0;
1514
1515         err = ufshcd_read_desc(hba, QUERY_DESC_IDN_STRING, desc_index, buf,
1516                                size);
1517
1518         if (err) {
1519                 dev_err(hba->dev, "%s: reading String Desc failed after %d retries. err = %d\n",
1520                         __func__, QUERY_REQ_RETRIES, err);
1521                 goto out;
1522         }
1523
1524         if (ascii) {
1525                 int desc_len;
1526                 int ascii_len;
1527                 int i;
1528                 u8 *buff_ascii;
1529
1530                 desc_len = buf[0];
1531                 /* remove header and divide by 2 to move from UTF16 to UTF8 */
1532                 ascii_len = (desc_len - QUERY_DESC_HDR_SIZE) / 2 + 1;
1533                 if (size < ascii_len + QUERY_DESC_HDR_SIZE) {
1534                         dev_err(hba->dev, "%s: buffer allocated size is too small\n",
1535                                 __func__);
1536                         err = -ENOMEM;
1537                         goto out;
1538                 }
1539
1540                 buff_ascii = kmalloc(ascii_len, GFP_KERNEL);
1541                 if (!buff_ascii) {
1542                         err = -ENOMEM;
1543                         goto out;
1544                 }
1545
1546                 /*
1547                  * the descriptor contains string in UTF16 format
1548                  * we need to convert to utf-8 so it can be displayed
1549                  */
1550                 utf16_to_utf8(buff_ascii,
1551                               (uint16_t *)&buf[QUERY_DESC_HDR_SIZE], ascii_len);
1552
1553                 /* replace non-printable or non-ASCII characters with spaces */
1554                 for (i = 0; i < ascii_len; i++)
1555                         ufshcd_remove_non_printable(&buff_ascii[i]);
1556
1557                 memset(buf + QUERY_DESC_HDR_SIZE, 0,
1558                        size - QUERY_DESC_HDR_SIZE);
1559                 memcpy(buf + QUERY_DESC_HDR_SIZE, buff_ascii, ascii_len);
1560                 buf[QUERY_DESC_LENGTH_OFFSET] = ascii_len + QUERY_DESC_HDR_SIZE;
1561                 kfree(buff_ascii);
1562         }
1563 out:
1564         return err;
1565 }
1566
1567 static int ufs_get_device_desc(struct ufs_hba *hba,
1568                                struct ufs_dev_desc *dev_desc)
1569 {
1570         int err;
1571         size_t buff_len;
1572         u8 model_index;
1573         u8 *desc_buf;
1574
1575         buff_len = max_t(size_t, hba->desc_size.dev_desc,
1576                          QUERY_DESC_MAX_SIZE + 1);
1577         desc_buf = kmalloc(buff_len, GFP_KERNEL);
1578         if (!desc_buf) {
1579                 err = -ENOMEM;
1580                 goto out;
1581         }
1582
1583         err = ufshcd_read_device_desc(hba, desc_buf, hba->desc_size.dev_desc);
1584         if (err) {
1585                 dev_err(hba->dev, "%s: Failed reading Device Desc. err = %d\n",
1586                         __func__, err);
1587                 goto out;
1588         }
1589
1590         /*
1591          * getting vendor (manufacturerID) and Bank Index in big endian
1592          * format
1593          */
1594         dev_desc->wmanufacturerid = desc_buf[DEVICE_DESC_PARAM_MANF_ID] << 8 |
1595                                      desc_buf[DEVICE_DESC_PARAM_MANF_ID + 1];
1596
1597         model_index = desc_buf[DEVICE_DESC_PARAM_PRDCT_NAME];
1598
1599         /* Zero-pad entire buffer for string termination. */
1600         memset(desc_buf, 0, buff_len);
1601
1602         err = ufshcd_read_string_desc(hba, model_index, desc_buf,
1603                                       QUERY_DESC_MAX_SIZE, true/*ASCII*/);
1604         if (err) {
1605                 dev_err(hba->dev, "%s: Failed reading Product Name. err = %d\n",
1606                         __func__, err);
1607                 goto out;
1608         }
1609
1610         desc_buf[QUERY_DESC_MAX_SIZE] = '\0';
1611         strlcpy(dev_desc->model, (char *)(desc_buf + QUERY_DESC_HDR_SIZE),
1612                 min_t(u8, desc_buf[QUERY_DESC_LENGTH_OFFSET],
1613                       MAX_MODEL_LEN));
1614
1615         /* Null terminate the model string */
1616         dev_desc->model[MAX_MODEL_LEN] = '\0';
1617
1618 out:
1619         kfree(desc_buf);
1620         return err;
1621 }
1622
1623 /**
1624  * ufshcd_get_max_pwr_mode - reads the max power mode negotiated with device
1625  */
1626 static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba)
1627 {
1628         struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info;
1629
1630         if (hba->max_pwr_info.is_valid)
1631                 return 0;
1632
1633         pwr_info->pwr_tx = FAST_MODE;
1634         pwr_info->pwr_rx = FAST_MODE;
1635         pwr_info->hs_rate = PA_HS_MODE_B;
1636
1637         /* Get the connected lane count */
1638         ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDRXDATALANES),
1639                        &pwr_info->lane_rx);
1640         ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
1641                        &pwr_info->lane_tx);
1642
1643         if (!pwr_info->lane_rx || !pwr_info->lane_tx) {
1644                 dev_err(hba->dev, "%s: invalid connected lanes value. rx=%d, tx=%d\n",
1645                         __func__, pwr_info->lane_rx, pwr_info->lane_tx);
1646                 return -EINVAL;
1647         }
1648
1649         /*
1650          * First, get the maximum gears of HS speed.
1651          * If a zero value, it means there is no HSGEAR capability.
1652          * Then, get the maximum gears of PWM speed.
1653          */
1654         ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), &pwr_info->gear_rx);
1655         if (!pwr_info->gear_rx) {
1656                 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
1657                                &pwr_info->gear_rx);
1658                 if (!pwr_info->gear_rx) {
1659                         dev_err(hba->dev, "%s: invalid max pwm rx gear read = %d\n",
1660                                 __func__, pwr_info->gear_rx);
1661                         return -EINVAL;
1662                 }
1663                 pwr_info->pwr_rx = SLOW_MODE;
1664         }
1665
1666         ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR),
1667                             &pwr_info->gear_tx);
1668         if (!pwr_info->gear_tx) {
1669                 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
1670                                     &pwr_info->gear_tx);
1671                 if (!pwr_info->gear_tx) {
1672                         dev_err(hba->dev, "%s: invalid max pwm tx gear read = %d\n",
1673                                 __func__, pwr_info->gear_tx);
1674                         return -EINVAL;
1675                 }
1676                 pwr_info->pwr_tx = SLOW_MODE;
1677         }
1678
1679         hba->max_pwr_info.is_valid = true;
1680         return 0;
1681 }
1682
1683 static int ufshcd_change_power_mode(struct ufs_hba *hba,
1684                                     struct ufs_pa_layer_attr *pwr_mode)
1685 {
1686         int ret;
1687
1688         /* if already configured to the requested pwr_mode */
1689         if (pwr_mode->gear_rx == hba->pwr_info.gear_rx &&
1690             pwr_mode->gear_tx == hba->pwr_info.gear_tx &&
1691             pwr_mode->lane_rx == hba->pwr_info.lane_rx &&
1692             pwr_mode->lane_tx == hba->pwr_info.lane_tx &&
1693             pwr_mode->pwr_rx == hba->pwr_info.pwr_rx &&
1694             pwr_mode->pwr_tx == hba->pwr_info.pwr_tx &&
1695             pwr_mode->hs_rate == hba->pwr_info.hs_rate) {
1696                 dev_dbg(hba->dev, "%s: power already configured\n", __func__);
1697                 return 0;
1698         }
1699
1700         /*
1701          * Configure attributes for power mode change with below.
1702          * - PA_RXGEAR, PA_ACTIVERXDATALANES, PA_RXTERMINATION,
1703          * - PA_TXGEAR, PA_ACTIVETXDATALANES, PA_TXTERMINATION,
1704          * - PA_HSSERIES
1705          */
1706         ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXGEAR), pwr_mode->gear_rx);
1707         ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVERXDATALANES),
1708                        pwr_mode->lane_rx);
1709         if (pwr_mode->pwr_rx == FASTAUTO_MODE || pwr_mode->pwr_rx == FAST_MODE)
1710                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), TRUE);
1711         else
1712                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), FALSE);
1713
1714         ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXGEAR), pwr_mode->gear_tx);
1715         ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVETXDATALANES),
1716                        pwr_mode->lane_tx);
1717         if (pwr_mode->pwr_tx == FASTAUTO_MODE || pwr_mode->pwr_tx == FAST_MODE)
1718                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), TRUE);
1719         else
1720                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), FALSE);
1721
1722         if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
1723             pwr_mode->pwr_tx == FASTAUTO_MODE ||
1724             pwr_mode->pwr_rx == FAST_MODE ||
1725             pwr_mode->pwr_tx == FAST_MODE)
1726                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HSSERIES),
1727                                pwr_mode->hs_rate);
1728
1729         ret = ufshcd_uic_change_pwr_mode(hba, pwr_mode->pwr_rx << 4 |
1730                                          pwr_mode->pwr_tx);
1731
1732         if (ret) {
1733                 dev_err(hba->dev,
1734                         "%s: power mode change failed %d\n", __func__, ret);
1735
1736                 return ret;
1737         }
1738
1739         /* Copy new Power Mode to power info */
1740         memcpy(&hba->pwr_info, pwr_mode, sizeof(struct ufs_pa_layer_attr));
1741
1742         return ret;
1743 }
1744
1745 /**
1746  * ufshcd_verify_dev_init() - Verify device initialization
1747  *
1748  */
1749 static int ufshcd_verify_dev_init(struct ufs_hba *hba)
1750 {
1751         int retries;
1752         int err;
1753
1754         for (retries = NOP_OUT_RETRIES; retries > 0; retries--) {
1755                 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_NOP,
1756                                           NOP_OUT_TIMEOUT);
1757                 if (!err || err == -ETIMEDOUT)
1758                         break;
1759
1760                 dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err);
1761         }
1762
1763         if (err)
1764                 dev_err(hba->dev, "%s: NOP OUT failed %d\n", __func__, err);
1765
1766         return err;
1767 }
1768
1769 /**
1770  * ufshcd_complete_dev_init() - checks device readiness
1771  */
1772 static int ufshcd_complete_dev_init(struct ufs_hba *hba)
1773 {
1774         int i;
1775         int err;
1776         bool flag_res = 1;
1777
1778         err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
1779                                       QUERY_FLAG_IDN_FDEVICEINIT, NULL);
1780         if (err) {
1781                 dev_err(hba->dev,
1782                         "%s setting fDeviceInit flag failed with error %d\n",
1783                         __func__, err);
1784                 goto out;
1785         }
1786
1787         /* poll for max. 1000 iterations for fDeviceInit flag to clear */
1788         for (i = 0; i < 1000 && !err && flag_res; i++)
1789                 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG,
1790                                               QUERY_FLAG_IDN_FDEVICEINIT,
1791                                               &flag_res);
1792
1793         if (err)
1794                 dev_err(hba->dev,
1795                         "%s reading fDeviceInit flag failed with error %d\n",
1796                         __func__, err);
1797         else if (flag_res)
1798                 dev_err(hba->dev,
1799                         "%s fDeviceInit was not cleared by the device\n",
1800                         __func__);
1801
1802 out:
1803         return err;
1804 }
1805
1806 static void ufshcd_def_desc_sizes(struct ufs_hba *hba)
1807 {
1808         hba->desc_size.dev_desc = QUERY_DESC_DEVICE_DEF_SIZE;
1809         hba->desc_size.pwr_desc = QUERY_DESC_POWER_DEF_SIZE;
1810         hba->desc_size.interc_desc = QUERY_DESC_INTERCONNECT_DEF_SIZE;
1811         hba->desc_size.conf_desc = QUERY_DESC_CONFIGURATION_DEF_SIZE;
1812         hba->desc_size.unit_desc = QUERY_DESC_UNIT_DEF_SIZE;
1813         hba->desc_size.geom_desc = QUERY_DESC_GEOMETRY_DEF_SIZE;
1814         hba->desc_size.hlth_desc = QUERY_DESC_HEALTH_DEF_SIZE;
1815 }
1816
1817 int ufs_start(struct ufs_hba *hba)
1818 {
1819         struct ufs_dev_desc card = {0};
1820         int ret;
1821
1822         ret = ufshcd_link_startup(hba);
1823         if (ret)
1824                 return ret;
1825
1826         ret = ufshcd_verify_dev_init(hba);
1827         if (ret)
1828                 return ret;
1829
1830         ret = ufshcd_complete_dev_init(hba);
1831         if (ret)
1832                 return ret;
1833
1834         /* Init check for device descriptor sizes */
1835         ufshcd_init_desc_sizes(hba);
1836
1837         ret = ufs_get_device_desc(hba, &card);
1838         if (ret) {
1839                 dev_err(hba->dev, "%s: Failed getting device info. err = %d\n",
1840                         __func__, ret);
1841
1842                 return ret;
1843         }
1844
1845         if (ufshcd_get_max_pwr_mode(hba)) {
1846                 dev_err(hba->dev,
1847                         "%s: Failed getting max supported power mode\n",
1848                         __func__);
1849         } else {
1850                 ret = ufshcd_change_power_mode(hba, &hba->max_pwr_info.info);
1851                 if (ret) {
1852                         dev_err(hba->dev, "%s: Failed setting power mode, err = %d\n",
1853                                 __func__, ret);
1854
1855                         return ret;
1856                 }
1857
1858                 printf("Device at %s up at:", hba->dev->name);
1859                 ufshcd_print_pwr_info(hba);
1860         }
1861
1862         return 0;
1863 }
1864
1865 int ufshcd_probe(struct udevice *ufs_dev, struct ufs_hba_ops *hba_ops)
1866 {
1867         struct ufs_hba *hba = dev_get_uclass_priv(ufs_dev);
1868         struct scsi_plat *scsi_plat;
1869         struct udevice *scsi_dev;
1870         int err;
1871
1872         device_find_first_child(ufs_dev, &scsi_dev);
1873         if (!scsi_dev)
1874                 return -ENODEV;
1875
1876         scsi_plat = dev_get_uclass_plat(scsi_dev);
1877         scsi_plat->max_id = UFSHCD_MAX_ID;
1878         scsi_plat->max_lun = UFS_MAX_LUNS;
1879         scsi_plat->max_bytes_per_req = UFS_MAX_BYTES;
1880
1881         hba->dev = ufs_dev;
1882         hba->ops = hba_ops;
1883         hba->mmio_base = (void *)dev_read_addr(ufs_dev);
1884
1885         /* Set descriptor lengths to specification defaults */
1886         ufshcd_def_desc_sizes(hba);
1887
1888         ufshcd_ops_init(hba);
1889
1890         /* Read capabilties registers */
1891         hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES);
1892
1893         /* Get UFS version supported by the controller */
1894         hba->version = ufshcd_get_ufs_version(hba);
1895         if (hba->version != UFSHCI_VERSION_10 &&
1896             hba->version != UFSHCI_VERSION_11 &&
1897             hba->version != UFSHCI_VERSION_20 &&
1898             hba->version != UFSHCI_VERSION_21)
1899                 dev_err(hba->dev, "invalid UFS version 0x%x\n",
1900                         hba->version);
1901
1902         /* Get Interrupt bit mask per version */
1903         hba->intr_mask = ufshcd_get_intr_mask(hba);
1904
1905         /* Allocate memory for host memory space */
1906         err = ufshcd_memory_alloc(hba);
1907         if (err) {
1908                 dev_err(hba->dev, "Memory allocation failed\n");
1909                 return err;
1910         }
1911
1912         /* Configure Local data structures */
1913         ufshcd_host_memory_configure(hba);
1914
1915         /*
1916          * In order to avoid any spurious interrupt immediately after
1917          * registering UFS controller interrupt handler, clear any pending UFS
1918          * interrupt status and disable all the UFS interrupts.
1919          */
1920         ufshcd_writel(hba, ufshcd_readl(hba, REG_INTERRUPT_STATUS),
1921                       REG_INTERRUPT_STATUS);
1922         ufshcd_writel(hba, 0, REG_INTERRUPT_ENABLE);
1923
1924         err = ufshcd_hba_enable(hba);
1925         if (err) {
1926                 dev_err(hba->dev, "Host controller enable failed\n");
1927                 return err;
1928         }
1929
1930         err = ufs_start(hba);
1931         if (err)
1932                 return err;
1933
1934         return 0;
1935 }
1936
1937 int ufs_scsi_bind(struct udevice *ufs_dev, struct udevice **scsi_devp)
1938 {
1939         int ret = device_bind_driver(ufs_dev, "ufs_scsi", "ufs_scsi",
1940                                      scsi_devp);
1941
1942         return ret;
1943 }
1944
1945 static struct scsi_ops ufs_ops = {
1946         .exec           = ufs_scsi_exec,
1947 };
1948
1949 int ufs_probe_dev(int index)
1950 {
1951         struct udevice *dev;
1952
1953         return uclass_get_device(UCLASS_UFS, index, &dev);
1954 }
1955
1956 int ufs_probe(void)
1957 {
1958         struct udevice *dev;
1959         int ret, i;
1960
1961         for (i = 0;; i++) {
1962                 ret = uclass_get_device(UCLASS_UFS, i, &dev);
1963                 if (ret == -ENODEV)
1964                         break;
1965         }
1966
1967         return 0;
1968 }
1969
1970 U_BOOT_DRIVER(ufs_scsi) = {
1971         .id = UCLASS_SCSI,
1972         .name = "ufs_scsi",
1973         .ops = &ufs_ops,
1974 };