--- /dev/null
+// SPDX-License-Identifier: GPL-2.0+
+/**
+ * ufs.c - Universal Flash Subsystem (UFS) driver
+ *
+ * Taken from Linux Kernel v5.2 (drivers/scsi/ufs/ufshcd.c) and ported
+ * to u-boot.
+ *
+ * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com
+ */
+
+#include <charset.h>
+#include <common.h>
+#include <dm.h>
+#include <dm/device_compat.h>
+#include <dm/devres.h>
+#include <dm/lists.h>
+#include <dm/device-internal.h>
+#include <malloc.h>
+#include <hexdump.h>
+#include <scsi.h>
+
+#include <linux/dma-mapping.h>
+
+#include "ufs.h"
+
+#define UFSHCD_ENABLE_INTRS (UTP_TRANSFER_REQ_COMPL |\
+ UTP_TASK_REQ_COMPL |\
+ UFSHCD_ERROR_MASK)
+/* maximum number of link-startup retries */
+#define DME_LINKSTARTUP_RETRIES 3
+
+/* maximum number of retries for a general UIC command */
+#define UFS_UIC_COMMAND_RETRIES 3
+
+/* Query request retries */
+#define QUERY_REQ_RETRIES 3
+/* Query request timeout */
+#define QUERY_REQ_TIMEOUT 1500 /* 1.5 seconds */
+
+/* maximum timeout in ms for a general UIC command */
+#define UFS_UIC_CMD_TIMEOUT 1000
+/* NOP OUT retries waiting for NOP IN response */
+#define NOP_OUT_RETRIES 10
+/* Timeout after 30 msecs if NOP OUT hangs without response */
+#define NOP_OUT_TIMEOUT 30 /* msecs */
+
+/* Only use one Task Tag for all requests */
+#define TASK_TAG 0
+
+/* Expose the flag value from utp_upiu_query.value */
+#define MASK_QUERY_UPIU_FLAG_LOC 0xFF
+
+#define MAX_PRDT_ENTRY 262144
+
+/* maximum bytes per request */
+#define UFS_MAX_BYTES (128 * 256 * 1024)
+
+static inline bool ufshcd_is_hba_active(struct ufs_hba *hba);
+static inline void ufshcd_hba_stop(struct ufs_hba *hba);
+static int ufshcd_hba_enable(struct ufs_hba *hba);
+
+/*
+ * ufshcd_wait_for_register - wait for register value to change
+ */
+static int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask,
+ u32 val, unsigned long timeout_ms)
+{
+ int err = 0;
+ unsigned long start = get_timer(0);
+
+ /* ignore bits that we don't intend to wait on */
+ val = val & mask;
+
+ while ((ufshcd_readl(hba, reg) & mask) != val) {
+ if (get_timer(start) > timeout_ms) {
+ if ((ufshcd_readl(hba, reg) & mask) != val)
+ err = -ETIMEDOUT;
+ break;
+ }
+ }
+
+ return err;
+}
+
+/**
+ * ufshcd_init_pwr_info - setting the POR (power on reset)
+ * values in hba power info
+ */
+static void ufshcd_init_pwr_info(struct ufs_hba *hba)
+{
+ hba->pwr_info.gear_rx = UFS_PWM_G1;
+ hba->pwr_info.gear_tx = UFS_PWM_G1;
+ hba->pwr_info.lane_rx = 1;
+ hba->pwr_info.lane_tx = 1;
+ hba->pwr_info.pwr_rx = SLOWAUTO_MODE;
+ hba->pwr_info.pwr_tx = SLOWAUTO_MODE;
+ hba->pwr_info.hs_rate = 0;
+}
+
+/**
+ * ufshcd_print_pwr_info - print power params as saved in hba
+ * power info
+ */
+static void ufshcd_print_pwr_info(struct ufs_hba *hba)
+{
+ static const char * const names[] = {
+ "INVALID MODE",
+ "FAST MODE",
+ "SLOW_MODE",
+ "INVALID MODE",
+ "FASTAUTO_MODE",
+ "SLOWAUTO_MODE",
+ "INVALID MODE",
+ };
+
+ dev_err(hba->dev, "[RX, TX]: gear=[%d, %d], lane[%d, %d], pwr[%s, %s], rate = %d\n",
+ hba->pwr_info.gear_rx, hba->pwr_info.gear_tx,
+ hba->pwr_info.lane_rx, hba->pwr_info.lane_tx,
+ names[hba->pwr_info.pwr_rx],
+ names[hba->pwr_info.pwr_tx],
+ hba->pwr_info.hs_rate);
+}
+
+/**
+ * ufshcd_ready_for_uic_cmd - Check if controller is ready
+ * to accept UIC commands
+ */
+static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba)
+{
+ if (ufshcd_readl(hba, REG_CONTROLLER_STATUS) & UIC_COMMAND_READY)
+ return true;
+ else
+ return false;
+}
+
+/**
+ * ufshcd_get_uic_cmd_result - Get the UIC command result
+ */
+static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba)
+{
+ return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2) &
+ MASK_UIC_COMMAND_RESULT;
+}
+
+/**
+ * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command
+ */
+static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba)
+{
+ return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3);
+}
+
+/**
+ * ufshcd_is_device_present - Check if any device connected to
+ * the host controller
+ */
+static inline bool ufshcd_is_device_present(struct ufs_hba *hba)
+{
+ return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) &
+ DEVICE_PRESENT) ? true : false;
+}
+
+/**
+ * ufshcd_send_uic_cmd - UFS Interconnect layer command API
+ *
+ */
+static int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
+{
+ unsigned long start = 0;
+ u32 intr_status;
+ u32 enabled_intr_status;
+
+ if (!ufshcd_ready_for_uic_cmd(hba)) {
+ dev_err(hba->dev,
+ "Controller not ready to accept UIC commands\n");
+ return -EIO;
+ }
+
+ debug("sending uic command:%d\n", uic_cmd->command);
+
+ /* Write Args */
+ ufshcd_writel(hba, uic_cmd->argument1, REG_UIC_COMMAND_ARG_1);
+ ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2);
+ ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3);
+
+ /* Write UIC Cmd */
+ ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK,
+ REG_UIC_COMMAND);
+
+ start = get_timer(0);
+ do {
+ intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
+ enabled_intr_status = intr_status & hba->intr_mask;
+ ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS);
+
+ if (get_timer(start) > UFS_UIC_CMD_TIMEOUT) {
+ dev_err(hba->dev,
+ "Timedout waiting for UIC response\n");
+
+ return -ETIMEDOUT;
+ }
+
+ if (enabled_intr_status & UFSHCD_ERROR_MASK) {
+ dev_err(hba->dev, "Error in status:%08x\n",
+ enabled_intr_status);
+
+ return -1;
+ }
+ } while (!(enabled_intr_status & UFSHCD_UIC_MASK));
+
+ uic_cmd->argument2 = ufshcd_get_uic_cmd_result(hba);
+ uic_cmd->argument3 = ufshcd_get_dme_attr_val(hba);
+
+ debug("Sent successfully\n");
+
+ return 0;
+}
+
+/**
+ * ufshcd_dme_set_attr - UIC command for DME_SET, DME_PEER_SET
+ *
+ */
+int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel, u8 attr_set,
+ u32 mib_val, u8 peer)
+{
+ struct uic_command uic_cmd = {0};
+ static const char *const action[] = {
+ "dme-set",
+ "dme-peer-set"
+ };
+ const char *set = action[!!peer];
+ int ret;
+ int retries = UFS_UIC_COMMAND_RETRIES;
+
+ uic_cmd.command = peer ?
+ UIC_CMD_DME_PEER_SET : UIC_CMD_DME_SET;
+ uic_cmd.argument1 = attr_sel;
+ uic_cmd.argument2 = UIC_ARG_ATTR_TYPE(attr_set);
+ uic_cmd.argument3 = mib_val;
+
+ do {
+ /* for peer attributes we retry upon failure */
+ ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
+ if (ret)
+ dev_dbg(hba->dev, "%s: attr-id 0x%x val 0x%x error code %d\n",
+ set, UIC_GET_ATTR_ID(attr_sel), mib_val, ret);
+ } while (ret && peer && --retries);
+
+ if (ret)
+ dev_err(hba->dev, "%s: attr-id 0x%x val 0x%x failed %d retries\n",
+ set, UIC_GET_ATTR_ID(attr_sel), mib_val,
+ UFS_UIC_COMMAND_RETRIES - retries);
+
+ return ret;
+}
+
+/**
+ * ufshcd_dme_get_attr - UIC command for DME_GET, DME_PEER_GET
+ *
+ */
+int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel,
+ u32 *mib_val, u8 peer)
+{
+ struct uic_command uic_cmd = {0};
+ static const char *const action[] = {
+ "dme-get",
+ "dme-peer-get"
+ };
+ const char *get = action[!!peer];
+ int ret;
+ int retries = UFS_UIC_COMMAND_RETRIES;
+
+ uic_cmd.command = peer ?
+ UIC_CMD_DME_PEER_GET : UIC_CMD_DME_GET;
+ uic_cmd.argument1 = attr_sel;
+
+ do {
+ /* for peer attributes we retry upon failure */
+ ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
+ if (ret)
+ dev_dbg(hba->dev, "%s: attr-id 0x%x error code %d\n",
+ get, UIC_GET_ATTR_ID(attr_sel), ret);
+ } while (ret && peer && --retries);
+
+ if (ret)
+ dev_err(hba->dev, "%s: attr-id 0x%x failed %d retries\n",
+ get, UIC_GET_ATTR_ID(attr_sel),
+ UFS_UIC_COMMAND_RETRIES - retries);
+
+ if (mib_val && !ret)
+ *mib_val = uic_cmd.argument3;
+
+ return ret;
+}
+
+static int ufshcd_disable_tx_lcc(struct ufs_hba *hba, bool peer)
+{
+ u32 tx_lanes, i, err = 0;
+
+ if (!peer)
+ ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
+ &tx_lanes);
+ else
+ ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
+ &tx_lanes);
+ for (i = 0; i < tx_lanes; i++) {
+ if (!peer)
+ err = ufshcd_dme_set(hba,
+ UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
+ UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
+ 0);
+ else
+ err = ufshcd_dme_peer_set(hba,
+ UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
+ UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
+ 0);
+ if (err) {
+ dev_err(hba->dev, "%s: TX LCC Disable failed, peer = %d, lane = %d, err = %d",
+ __func__, peer, i, err);
+ break;
+ }
+ }
+
+ return err;
+}
+
+static inline int ufshcd_disable_device_tx_lcc(struct ufs_hba *hba)
+{
+ return ufshcd_disable_tx_lcc(hba, true);
+}
+
+/**
+ * ufshcd_dme_link_startup - Notify Unipro to perform link startup
+ *
+ */
+static int ufshcd_dme_link_startup(struct ufs_hba *hba)
+{
+ struct uic_command uic_cmd = {0};
+ int ret;
+
+ uic_cmd.command = UIC_CMD_DME_LINK_STARTUP;
+
+ ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
+ if (ret)
+ dev_dbg(hba->dev,
+ "dme-link-startup: error code %d\n", ret);
+ return ret;
+}
+
+/**
+ * ufshcd_disable_intr_aggr - Disables interrupt aggregation.
+ *
+ */
+static inline void ufshcd_disable_intr_aggr(struct ufs_hba *hba)
+{
+ ufshcd_writel(hba, 0, REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
+}
+
+/**
+ * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY
+ */
+static inline int ufshcd_get_lists_status(u32 reg)
+{
+ return !((reg & UFSHCD_STATUS_READY) == UFSHCD_STATUS_READY);
+}
+
+/**
+ * ufshcd_enable_run_stop_reg - Enable run-stop registers,
+ * When run-stop registers are set to 1, it indicates the
+ * host controller that it can process the requests
+ */
+static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba)
+{
+ ufshcd_writel(hba, UTP_TASK_REQ_LIST_RUN_STOP_BIT,
+ REG_UTP_TASK_REQ_LIST_RUN_STOP);
+ ufshcd_writel(hba, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT,
+ REG_UTP_TRANSFER_REQ_LIST_RUN_STOP);
+}
+
+/**
+ * ufshcd_enable_intr - enable interrupts
+ */
+static void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs)
+{
+ u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
+ u32 rw;
+
+ if (hba->version == UFSHCI_VERSION_10) {
+ rw = set & INTERRUPT_MASK_RW_VER_10;
+ set = rw | ((set ^ intrs) & intrs);
+ } else {
+ set |= intrs;
+ }
+
+ ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
+
+ hba->intr_mask = set;
+}
+
+/**
+ * ufshcd_make_hba_operational - Make UFS controller operational
+ *
+ * To bring UFS host controller to operational state,
+ * 1. Enable required interrupts
+ * 2. Configure interrupt aggregation
+ * 3. Program UTRL and UTMRL base address
+ * 4. Configure run-stop-registers
+ *
+ */
+static int ufshcd_make_hba_operational(struct ufs_hba *hba)
+{
+ int err = 0;
+ u32 reg;
+
+ /* Enable required interrupts */
+ ufshcd_enable_intr(hba, UFSHCD_ENABLE_INTRS);
+
+ /* Disable interrupt aggregation */
+ ufshcd_disable_intr_aggr(hba);
+
+ /* Configure UTRL and UTMRL base address registers */
+ ufshcd_writel(hba, lower_32_bits((dma_addr_t)hba->utrdl),
+ REG_UTP_TRANSFER_REQ_LIST_BASE_L);
+ ufshcd_writel(hba, upper_32_bits((dma_addr_t)hba->utrdl),
+ REG_UTP_TRANSFER_REQ_LIST_BASE_H);
+ ufshcd_writel(hba, lower_32_bits((dma_addr_t)hba->utmrdl),
+ REG_UTP_TASK_REQ_LIST_BASE_L);
+ ufshcd_writel(hba, upper_32_bits((dma_addr_t)hba->utmrdl),
+ REG_UTP_TASK_REQ_LIST_BASE_H);
+
+ /*
+ * UCRDY, UTMRLDY and UTRLRDY bits must be 1
+ */
+ reg = ufshcd_readl(hba, REG_CONTROLLER_STATUS);
+ if (!(ufshcd_get_lists_status(reg))) {
+ ufshcd_enable_run_stop_reg(hba);
+ } else {
+ dev_err(hba->dev,
+ "Host controller not ready to process requests");
+ err = -EIO;
+ goto out;
+ }
+
+out:
+ return err;
+}
+
+/**
+ * ufshcd_link_startup - Initialize unipro link startup
+ */
+static int ufshcd_link_startup(struct ufs_hba *hba)
+{
+ int ret;
+ int retries = DME_LINKSTARTUP_RETRIES;
+ bool link_startup_again = true;
+
+link_startup:
+ do {
+ ufshcd_ops_link_startup_notify(hba, PRE_CHANGE);
+
+ ret = ufshcd_dme_link_startup(hba);
+
+ /* check if device is detected by inter-connect layer */
+ if (!ret && !ufshcd_is_device_present(hba)) {
+ dev_err(hba->dev, "%s: Device not present\n", __func__);
+ ret = -ENXIO;
+ goto out;
+ }
+
+ /*
+ * DME link lost indication is only received when link is up,
+ * but we can't be sure if the link is up until link startup
+ * succeeds. So reset the local Uni-Pro and try again.
+ */
+ if (ret && ufshcd_hba_enable(hba))
+ goto out;
+ } while (ret && retries--);
+
+ if (ret)
+ /* failed to get the link up... retire */
+ goto out;
+
+ if (link_startup_again) {
+ link_startup_again = false;
+ retries = DME_LINKSTARTUP_RETRIES;
+ goto link_startup;
+ }
+
+ /* Mark that link is up in PWM-G1, 1-lane, SLOW-AUTO mode */
+ ufshcd_init_pwr_info(hba);
+
+ if (hba->quirks & UFSHCD_QUIRK_BROKEN_LCC) {
+ ret = ufshcd_disable_device_tx_lcc(hba);
+ if (ret)
+ goto out;
+ }
+
+ /* Include any host controller configuration via UIC commands */
+ ret = ufshcd_ops_link_startup_notify(hba, POST_CHANGE);
+ if (ret)
+ goto out;
+
+ ret = ufshcd_make_hba_operational(hba);
+out:
+ if (ret)
+ dev_err(hba->dev, "link startup failed %d\n", ret);
+
+ return ret;
+}
+
+/**
+ * ufshcd_hba_stop - Send controller to reset state
+ */
+static inline void ufshcd_hba_stop(struct ufs_hba *hba)
+{
+ int err;
+
+ ufshcd_writel(hba, CONTROLLER_DISABLE, REG_CONTROLLER_ENABLE);
+ err = ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE,
+ CONTROLLER_ENABLE, CONTROLLER_DISABLE,
+ 10);
+ if (err)
+ dev_err(hba->dev, "%s: Controller disable failed\n", __func__);
+}
+
+/**
+ * ufshcd_is_hba_active - Get controller state
+ */
+static inline bool ufshcd_is_hba_active(struct ufs_hba *hba)
+{
+ return (ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & CONTROLLER_ENABLE)
+ ? false : true;
+}
+
+/**
+ * ufshcd_hba_start - Start controller initialization sequence
+ */
+static inline void ufshcd_hba_start(struct ufs_hba *hba)
+{
+ ufshcd_writel(hba, CONTROLLER_ENABLE, REG_CONTROLLER_ENABLE);
+}
+
+/**
+ * ufshcd_hba_enable - initialize the controller
+ */
+static int ufshcd_hba_enable(struct ufs_hba *hba)
+{
+ int retry;
+
+ if (!ufshcd_is_hba_active(hba))
+ /* change controller state to "reset state" */
+ ufshcd_hba_stop(hba);
+
+ ufshcd_ops_hce_enable_notify(hba, PRE_CHANGE);
+
+ /* start controller initialization sequence */
+ ufshcd_hba_start(hba);
+
+ /*
+ * To initialize a UFS host controller HCE bit must be set to 1.
+ * During initialization the HCE bit value changes from 1->0->1.
+ * When the host controller completes initialization sequence
+ * it sets the value of HCE bit to 1. The same HCE bit is read back
+ * to check if the controller has completed initialization sequence.
+ * So without this delay the value HCE = 1, set in the previous
+ * instruction might be read back.
+ * This delay can be changed based on the controller.
+ */
+ mdelay(1);
+
+ /* wait for the host controller to complete initialization */
+ retry = 10;
+ while (ufshcd_is_hba_active(hba)) {
+ if (retry) {
+ retry--;
+ } else {
+ dev_err(hba->dev, "Controller enable failed\n");
+ return -EIO;
+ }
+ mdelay(5);
+ }
+
+ /* enable UIC related interrupts */
+ ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
+
+ ufshcd_ops_hce_enable_notify(hba, POST_CHANGE);
+
+ return 0;
+}
+
+/**
+ * ufshcd_host_memory_configure - configure local reference block with
+ * memory offsets
+ */
+static void ufshcd_host_memory_configure(struct ufs_hba *hba)
+{
+ struct utp_transfer_req_desc *utrdlp;
+ dma_addr_t cmd_desc_dma_addr;
+ u16 response_offset;
+ u16 prdt_offset;
+
+ utrdlp = hba->utrdl;
+ cmd_desc_dma_addr = (dma_addr_t)hba->ucdl;
+
+ utrdlp->command_desc_base_addr_lo =
+ cpu_to_le32(lower_32_bits(cmd_desc_dma_addr));
+ utrdlp->command_desc_base_addr_hi =
+ cpu_to_le32(upper_32_bits(cmd_desc_dma_addr));
+
+ response_offset = offsetof(struct utp_transfer_cmd_desc, response_upiu);
+ prdt_offset = offsetof(struct utp_transfer_cmd_desc, prd_table);
+
+ utrdlp->response_upiu_offset = cpu_to_le16(response_offset >> 2);
+ utrdlp->prd_table_offset = cpu_to_le16(prdt_offset >> 2);
+ utrdlp->response_upiu_length = cpu_to_le16(ALIGNED_UPIU_SIZE >> 2);
+
+ hba->ucd_req_ptr = (struct utp_upiu_req *)hba->ucdl;
+ hba->ucd_rsp_ptr =
+ (struct utp_upiu_rsp *)&hba->ucdl->response_upiu;
+ hba->ucd_prdt_ptr =
+ (struct ufshcd_sg_entry *)&hba->ucdl->prd_table;
+}
+
+/**
+ * ufshcd_memory_alloc - allocate memory for host memory space data structures
+ */
+static int ufshcd_memory_alloc(struct ufs_hba *hba)
+{
+ /* Allocate one Transfer Request Descriptor
+ * Should be aligned to 1k boundary.
+ */
+ hba->utrdl = memalign(1024, sizeof(struct utp_transfer_req_desc));
+ if (!hba->utrdl) {
+ dev_err(hba->dev, "Transfer Descriptor memory allocation failed\n");
+ return -ENOMEM;
+ }
+
+ /* Allocate one Command Descriptor
+ * Should be aligned to 1k boundary.
+ */
+ hba->ucdl = memalign(1024, sizeof(struct utp_transfer_cmd_desc));
+ if (!hba->ucdl) {
+ dev_err(hba->dev, "Command descriptor memory allocation failed\n");
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+/**
+ * ufshcd_get_intr_mask - Get the interrupt bit mask
+ */
+static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba)
+{
+ u32 intr_mask = 0;
+
+ switch (hba->version) {
+ case UFSHCI_VERSION_10:
+ intr_mask = INTERRUPT_MASK_ALL_VER_10;
+ break;
+ case UFSHCI_VERSION_11:
+ case UFSHCI_VERSION_20:
+ intr_mask = INTERRUPT_MASK_ALL_VER_11;
+ break;
+ case UFSHCI_VERSION_21:
+ default:
+ intr_mask = INTERRUPT_MASK_ALL_VER_21;
+ break;
+ }
+
+ return intr_mask;
+}
+
+/**
+ * ufshcd_get_ufs_version - Get the UFS version supported by the HBA
+ */
+static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba)
+{
+ return ufshcd_readl(hba, REG_UFS_VERSION);
+}
+
+/**
+ * ufshcd_get_upmcrs - Get the power mode change request status
+ */
+static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba)
+{
+ return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) >> 8) & 0x7;
+}
+
+/**
+ * ufshcd_prepare_req_desc_hdr() - Fills the requests header
+ * descriptor according to request
+ */
+static void ufshcd_prepare_req_desc_hdr(struct utp_transfer_req_desc *req_desc,
+ u32 *upiu_flags,
+ enum dma_data_direction cmd_dir)
+{
+ u32 data_direction;
+ u32 dword_0;
+
+ if (cmd_dir == DMA_FROM_DEVICE) {
+ data_direction = UTP_DEVICE_TO_HOST;
+ *upiu_flags = UPIU_CMD_FLAGS_READ;
+ } else if (cmd_dir == DMA_TO_DEVICE) {
+ data_direction = UTP_HOST_TO_DEVICE;
+ *upiu_flags = UPIU_CMD_FLAGS_WRITE;
+ } else {
+ data_direction = UTP_NO_DATA_TRANSFER;
+ *upiu_flags = UPIU_CMD_FLAGS_NONE;
+ }
+
+ dword_0 = data_direction | (0x1 << UPIU_COMMAND_TYPE_OFFSET);
+
+ /* Enable Interrupt for command */
+ dword_0 |= UTP_REQ_DESC_INT_CMD;
+
+ /* Transfer request descriptor header fields */
+ req_desc->header.dword_0 = cpu_to_le32(dword_0);
+ /* dword_1 is reserved, hence it is set to 0 */
+ req_desc->header.dword_1 = 0;
+ /*
+ * assigning invalid value for command status. Controller
+ * updates OCS on command completion, with the command
+ * status
+ */
+ req_desc->header.dword_2 =
+ cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
+ /* dword_3 is reserved, hence it is set to 0 */
+ req_desc->header.dword_3 = 0;
+
+ req_desc->prd_table_length = 0;
+}
+
+static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba,
+ u32 upiu_flags)
+{
+ struct utp_upiu_req *ucd_req_ptr = hba->ucd_req_ptr;
+ struct ufs_query *query = &hba->dev_cmd.query;
+ u16 len = be16_to_cpu(query->request.upiu_req.length);
+
+ /* Query request header */
+ ucd_req_ptr->header.dword_0 =
+ UPIU_HEADER_DWORD(UPIU_TRANSACTION_QUERY_REQ,
+ upiu_flags, 0, TASK_TAG);
+ ucd_req_ptr->header.dword_1 =
+ UPIU_HEADER_DWORD(0, query->request.query_func,
+ 0, 0);
+
+ /* Data segment length only need for WRITE_DESC */
+ if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
+ ucd_req_ptr->header.dword_2 =
+ UPIU_HEADER_DWORD(0, 0, (len >> 8), (u8)len);
+ else
+ ucd_req_ptr->header.dword_2 = 0;
+
+ /* Copy the Query Request buffer as is */
+ memcpy(&ucd_req_ptr->qr, &query->request.upiu_req, QUERY_OSF_SIZE);
+
+ /* Copy the Descriptor */
+ if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
+ memcpy(ucd_req_ptr + 1, query->descriptor, len);
+
+ memset(hba->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
+}
+
+static inline void ufshcd_prepare_utp_nop_upiu(struct ufs_hba *hba)
+{
+ struct utp_upiu_req *ucd_req_ptr = hba->ucd_req_ptr;
+
+ memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req));
+
+ /* command descriptor fields */
+ ucd_req_ptr->header.dword_0 =
+ UPIU_HEADER_DWORD(UPIU_TRANSACTION_NOP_OUT, 0, 0, 0x1f);
+ /* clear rest of the fields of basic header */
+ ucd_req_ptr->header.dword_1 = 0;
+ ucd_req_ptr->header.dword_2 = 0;
+
+ memset(hba->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
+}
+
+/**
+ * ufshcd_comp_devman_upiu - UFS Protocol Information Unit(UPIU)
+ * for Device Management Purposes
+ */
+static int ufshcd_comp_devman_upiu(struct ufs_hba *hba,
+ enum dev_cmd_type cmd_type)
+{
+ u32 upiu_flags;
+ int ret = 0;
+ struct utp_transfer_req_desc *req_desc = hba->utrdl;
+
+ hba->dev_cmd.type = cmd_type;
+
+ ufshcd_prepare_req_desc_hdr(req_desc, &upiu_flags, DMA_NONE);
+ switch (cmd_type) {
+ case DEV_CMD_TYPE_QUERY:
+ ufshcd_prepare_utp_query_req_upiu(hba, upiu_flags);
+ break;
+ case DEV_CMD_TYPE_NOP:
+ ufshcd_prepare_utp_nop_upiu(hba);
+ break;
+ default:
+ ret = -EINVAL;
+ }
+
+ return ret;
+}
+
+static int ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag)
+{
+ unsigned long start;
+ u32 intr_status;
+ u32 enabled_intr_status;
+
+ ufshcd_writel(hba, 1 << task_tag, REG_UTP_TRANSFER_REQ_DOOR_BELL);
+
+ start = get_timer(0);
+ do {
+ intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
+ enabled_intr_status = intr_status & hba->intr_mask;
+ ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS);
+
+ if (get_timer(start) > QUERY_REQ_TIMEOUT) {
+ dev_err(hba->dev,
+ "Timedout waiting for UTP response\n");
+
+ return -ETIMEDOUT;
+ }
+
+ if (enabled_intr_status & UFSHCD_ERROR_MASK) {
+ dev_err(hba->dev, "Error in status:%08x\n",
+ enabled_intr_status);
+
+ return -1;
+ }
+ } while (!(enabled_intr_status & UTP_TRANSFER_REQ_COMPL));
+
+ return 0;
+}
+
+/**
+ * ufshcd_get_req_rsp - returns the TR response transaction type
+ */
+static inline int ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr)
+{
+ return be32_to_cpu(ucd_rsp_ptr->header.dword_0) >> 24;
+}
+
+/**
+ * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status
+ *
+ */
+static inline int ufshcd_get_tr_ocs(struct ufs_hba *hba)
+{
+ return le32_to_cpu(hba->utrdl->header.dword_2) & MASK_OCS;
+}
+
+static inline int ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp *ucd_rsp_ptr)
+{
+ return be32_to_cpu(ucd_rsp_ptr->header.dword_1) & MASK_RSP_UPIU_RESULT;
+}
+
+static int ufshcd_check_query_response(struct ufs_hba *hba)
+{
+ struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
+
+ /* Get the UPIU response */
+ query_res->response = ufshcd_get_rsp_upiu_result(hba->ucd_rsp_ptr) >>
+ UPIU_RSP_CODE_OFFSET;
+ return query_res->response;
+}
+
+/**
+ * ufshcd_copy_query_response() - Copy the Query Response and the data
+ * descriptor
+ */
+static int ufshcd_copy_query_response(struct ufs_hba *hba)
+{
+ struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
+
+ memcpy(&query_res->upiu_res, &hba->ucd_rsp_ptr->qr, QUERY_OSF_SIZE);
+
+ /* Get the descriptor */
+ if (hba->dev_cmd.query.descriptor &&
+ hba->ucd_rsp_ptr->qr.opcode == UPIU_QUERY_OPCODE_READ_DESC) {
+ u8 *descp = (u8 *)hba->ucd_rsp_ptr +
+ GENERAL_UPIU_REQUEST_SIZE;
+ u16 resp_len;
+ u16 buf_len;
+
+ /* data segment length */
+ resp_len = be32_to_cpu(hba->ucd_rsp_ptr->header.dword_2) &
+ MASK_QUERY_DATA_SEG_LEN;
+ buf_len =
+ be16_to_cpu(hba->dev_cmd.query.request.upiu_req.length);
+ if (likely(buf_len >= resp_len)) {
+ memcpy(hba->dev_cmd.query.descriptor, descp, resp_len);
+ } else {
+ dev_warn(hba->dev,
+ "%s: Response size is bigger than buffer",
+ __func__);
+ return -EINVAL;
+ }
+ }
+
+ return 0;
+}
+
+/**
+ * ufshcd_exec_dev_cmd - API for sending device management requests
+ */
+static int ufshcd_exec_dev_cmd(struct ufs_hba *hba, enum dev_cmd_type cmd_type,
+ int timeout)
+{
+ int err;
+ int resp;
+
+ err = ufshcd_comp_devman_upiu(hba, cmd_type);
+ if (err)
+ return err;
+
+ err = ufshcd_send_command(hba, TASK_TAG);
+ if (err)
+ return err;
+
+ err = ufshcd_get_tr_ocs(hba);
+ if (err) {
+ dev_err(hba->dev, "Error in OCS:%d\n", err);
+ return -EINVAL;
+ }
+
+ resp = ufshcd_get_req_rsp(hba->ucd_rsp_ptr);
+ switch (resp) {
+ case UPIU_TRANSACTION_NOP_IN:
+ break;
+ case UPIU_TRANSACTION_QUERY_RSP:
+ err = ufshcd_check_query_response(hba);
+ if (!err)
+ err = ufshcd_copy_query_response(hba);
+ break;
+ case UPIU_TRANSACTION_REJECT_UPIU:
+ /* TODO: handle Reject UPIU Response */
+ err = -EPERM;
+ dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n",
+ __func__);
+ break;
+ default:
+ err = -EINVAL;
+ dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n",
+ __func__, resp);
+ }
+
+ return err;
+}
+
+/**
+ * ufshcd_init_query() - init the query response and request parameters
+ */
+static inline void ufshcd_init_query(struct ufs_hba *hba,
+ struct ufs_query_req **request,
+ struct ufs_query_res **response,
+ enum query_opcode opcode,
+ u8 idn, u8 index, u8 selector)
+{
+ *request = &hba->dev_cmd.query.request;
+ *response = &hba->dev_cmd.query.response;
+ memset(*request, 0, sizeof(struct ufs_query_req));
+ memset(*response, 0, sizeof(struct ufs_query_res));
+ (*request)->upiu_req.opcode = opcode;
+ (*request)->upiu_req.idn = idn;
+ (*request)->upiu_req.index = index;
+ (*request)->upiu_req.selector = selector;
+}
+
+/**
+ * ufshcd_query_flag() - API function for sending flag query requests
+ */
+int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode,
+ enum flag_idn idn, bool *flag_res)
+{
+ struct ufs_query_req *request = NULL;
+ struct ufs_query_res *response = NULL;
+ int err, index = 0, selector = 0;
+ int timeout = QUERY_REQ_TIMEOUT;
+
+ ufshcd_init_query(hba, &request, &response, opcode, idn, index,
+ selector);
+
+ switch (opcode) {
+ case UPIU_QUERY_OPCODE_SET_FLAG:
+ case UPIU_QUERY_OPCODE_CLEAR_FLAG:
+ case UPIU_QUERY_OPCODE_TOGGLE_FLAG:
+ request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
+ break;
+ case UPIU_QUERY_OPCODE_READ_FLAG:
+ request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
+ if (!flag_res) {
+ /* No dummy reads */
+ dev_err(hba->dev, "%s: Invalid argument for read request\n",
+ __func__);
+ err = -EINVAL;
+ goto out;
+ }
+ break;
+ default:
+ dev_err(hba->dev,
+ "%s: Expected query flag opcode but got = %d\n",
+ __func__, opcode);
+ err = -EINVAL;
+ goto out;
+ }
+
+ err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, timeout);
+
+ if (err) {
+ dev_err(hba->dev,
+ "%s: Sending flag query for idn %d failed, err = %d\n",
+ __func__, idn, err);
+ goto out;
+ }
+
+ if (flag_res)
+ *flag_res = (be32_to_cpu(response->upiu_res.value) &
+ MASK_QUERY_UPIU_FLAG_LOC) & 0x1;
+
+out:
+ return err;
+}
+
+static int ufshcd_query_flag_retry(struct ufs_hba *hba,
+ enum query_opcode opcode,
+ enum flag_idn idn, bool *flag_res)
+{
+ int ret;
+ int retries;
+
+ for (retries = 0; retries < QUERY_REQ_RETRIES; retries++) {
+ ret = ufshcd_query_flag(hba, opcode, idn, flag_res);
+ if (ret)
+ dev_dbg(hba->dev,
+ "%s: failed with error %d, retries %d\n",
+ __func__, ret, retries);
+ else
+ break;
+ }
+
+ if (ret)
+ dev_err(hba->dev,
+ "%s: query attribute, opcode %d, idn %d, failed with error %d after %d retires\n",
+ __func__, opcode, idn, ret, retries);
+ return ret;
+}
+
+static int __ufshcd_query_descriptor(struct ufs_hba *hba,
+ enum query_opcode opcode,
+ enum desc_idn idn, u8 index, u8 selector,
+ u8 *desc_buf, int *buf_len)
+{
+ struct ufs_query_req *request = NULL;
+ struct ufs_query_res *response = NULL;
+ int err;
+
+ if (!desc_buf) {
+ dev_err(hba->dev, "%s: descriptor buffer required for opcode 0x%x\n",
+ __func__, opcode);
+ err = -EINVAL;
+ goto out;
+ }
+
+ if (*buf_len < QUERY_DESC_MIN_SIZE || *buf_len > QUERY_DESC_MAX_SIZE) {
+ dev_err(hba->dev, "%s: descriptor buffer size (%d) is out of range\n",
+ __func__, *buf_len);
+ err = -EINVAL;
+ goto out;
+ }
+
+ ufshcd_init_query(hba, &request, &response, opcode, idn, index,
+ selector);
+ hba->dev_cmd.query.descriptor = desc_buf;
+ request->upiu_req.length = cpu_to_be16(*buf_len);
+
+ switch (opcode) {
+ case UPIU_QUERY_OPCODE_WRITE_DESC:
+ request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
+ break;
+ case UPIU_QUERY_OPCODE_READ_DESC:
+ request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
+ break;
+ default:
+ dev_err(hba->dev, "%s: Expected query descriptor opcode but got = 0x%.2x\n",
+ __func__, opcode);
+ err = -EINVAL;
+ goto out;
+ }
+
+ err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
+
+ if (err) {
+ dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
+ __func__, opcode, idn, index, err);
+ goto out;
+ }
+
+ hba->dev_cmd.query.descriptor = NULL;
+ *buf_len = be16_to_cpu(response->upiu_res.length);
+
+out:
+ return err;
+}
+
+/**
+ * ufshcd_query_descriptor_retry - API function for sending descriptor requests
+ */
+int ufshcd_query_descriptor_retry(struct ufs_hba *hba, enum query_opcode opcode,
+ enum desc_idn idn, u8 index, u8 selector,
+ u8 *desc_buf, int *buf_len)
+{
+ int err;
+ int retries;
+
+ for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
+ err = __ufshcd_query_descriptor(hba, opcode, idn, index,
+ selector, desc_buf, buf_len);
+ if (!err || err == -EINVAL)
+ break;
+ }
+
+ return err;
+}
+
+/**
+ * ufshcd_read_desc_length - read the specified descriptor length from header
+ */
+static int ufshcd_read_desc_length(struct ufs_hba *hba, enum desc_idn desc_id,
+ int desc_index, int *desc_length)
+{
+ int ret;
+ u8 header[QUERY_DESC_HDR_SIZE];
+ int header_len = QUERY_DESC_HDR_SIZE;
+
+ if (desc_id >= QUERY_DESC_IDN_MAX)
+ return -EINVAL;
+
+ ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC,
+ desc_id, desc_index, 0, header,
+ &header_len);
+
+ if (ret) {
+ dev_err(hba->dev, "%s: Failed to get descriptor header id %d",
+ __func__, desc_id);
+ return ret;
+ } else if (desc_id != header[QUERY_DESC_DESC_TYPE_OFFSET]) {
+ dev_warn(hba->dev, "%s: descriptor header id %d and desc_id %d mismatch",
+ __func__, header[QUERY_DESC_DESC_TYPE_OFFSET],
+ desc_id);
+ ret = -EINVAL;
+ }
+
+ *desc_length = header[QUERY_DESC_LENGTH_OFFSET];
+
+ return ret;
+}
+
+static void ufshcd_init_desc_sizes(struct ufs_hba *hba)
+{
+ int err;
+
+ err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_DEVICE, 0,
+ &hba->desc_size.dev_desc);
+ if (err)
+ hba->desc_size.dev_desc = QUERY_DESC_DEVICE_DEF_SIZE;
+
+ err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_POWER, 0,
+ &hba->desc_size.pwr_desc);
+ if (err)
+ hba->desc_size.pwr_desc = QUERY_DESC_POWER_DEF_SIZE;
+
+ err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_INTERCONNECT, 0,
+ &hba->desc_size.interc_desc);
+ if (err)
+ hba->desc_size.interc_desc = QUERY_DESC_INTERCONNECT_DEF_SIZE;
+
+ err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_CONFIGURATION, 0,
+ &hba->desc_size.conf_desc);
+ if (err)
+ hba->desc_size.conf_desc = QUERY_DESC_CONFIGURATION_DEF_SIZE;
+
+ err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_UNIT, 0,
+ &hba->desc_size.unit_desc);
+ if (err)
+ hba->desc_size.unit_desc = QUERY_DESC_UNIT_DEF_SIZE;
+
+ err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_GEOMETRY, 0,
+ &hba->desc_size.geom_desc);
+ if (err)
+ hba->desc_size.geom_desc = QUERY_DESC_GEOMETRY_DEF_SIZE;
+
+ err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_HEALTH, 0,
+ &hba->desc_size.hlth_desc);
+ if (err)
+ hba->desc_size.hlth_desc = QUERY_DESC_HEALTH_DEF_SIZE;
+}
+
+/**
+ * ufshcd_map_desc_id_to_length - map descriptor IDN to its length
+ *
+ */
+int ufshcd_map_desc_id_to_length(struct ufs_hba *hba, enum desc_idn desc_id,
+ int *desc_len)
+{
+ switch (desc_id) {
+ case QUERY_DESC_IDN_DEVICE:
+ *desc_len = hba->desc_size.dev_desc;
+ break;
+ case QUERY_DESC_IDN_POWER:
+ *desc_len = hba->desc_size.pwr_desc;
+ break;
+ case QUERY_DESC_IDN_GEOMETRY:
+ *desc_len = hba->desc_size.geom_desc;
+ break;
+ case QUERY_DESC_IDN_CONFIGURATION:
+ *desc_len = hba->desc_size.conf_desc;
+ break;
+ case QUERY_DESC_IDN_UNIT:
+ *desc_len = hba->desc_size.unit_desc;
+ break;
+ case QUERY_DESC_IDN_INTERCONNECT:
+ *desc_len = hba->desc_size.interc_desc;
+ break;
+ case QUERY_DESC_IDN_STRING:
+ *desc_len = QUERY_DESC_MAX_SIZE;
+ break;
+ case QUERY_DESC_IDN_HEALTH:
+ *desc_len = hba->desc_size.hlth_desc;
+ break;
+ case QUERY_DESC_IDN_RFU_0:
+ case QUERY_DESC_IDN_RFU_1:
+ *desc_len = 0;
+ break;
+ default:
+ *desc_len = 0;
+ return -EINVAL;
+ }
+ return 0;
+}
+EXPORT_SYMBOL(ufshcd_map_desc_id_to_length);
+
+/**
+ * ufshcd_read_desc_param - read the specified descriptor parameter
+ *
+ */
+int ufshcd_read_desc_param(struct ufs_hba *hba, enum desc_idn desc_id,
+ int desc_index, u8 param_offset, u8 *param_read_buf,
+ u8 param_size)
+{
+ int ret;
+ u8 *desc_buf;
+ int buff_len;
+ bool is_kmalloc = true;
+
+ /* Safety check */
+ if (desc_id >= QUERY_DESC_IDN_MAX || !param_size)
+ return -EINVAL;
+
+ /* Get the max length of descriptor from structure filled up at probe
+ * time.
+ */
+ ret = ufshcd_map_desc_id_to_length(hba, desc_id, &buff_len);
+
+ /* Sanity checks */
+ if (ret || !buff_len) {
+ dev_err(hba->dev, "%s: Failed to get full descriptor length",
+ __func__);
+ return ret;
+ }
+
+ /* Check whether we need temp memory */
+ if (param_offset != 0 || param_size < buff_len) {
+ desc_buf = kmalloc(buff_len, GFP_KERNEL);
+ if (!desc_buf)
+ return -ENOMEM;
+ } else {
+ desc_buf = param_read_buf;
+ is_kmalloc = false;
+ }
+
+ /* Request for full descriptor */
+ ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC,
+ desc_id, desc_index, 0, desc_buf,
+ &buff_len);
+
+ if (ret) {
+ dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d, desc_index %d, param_offset %d, ret %d",
+ __func__, desc_id, desc_index, param_offset, ret);
+ goto out;
+ }
+
+ /* Sanity check */
+ if (desc_buf[QUERY_DESC_DESC_TYPE_OFFSET] != desc_id) {
+ dev_err(hba->dev, "%s: invalid desc_id %d in descriptor header",
+ __func__, desc_buf[QUERY_DESC_DESC_TYPE_OFFSET]);
+ ret = -EINVAL;
+ goto out;
+ }
+
+ /* Check wherher we will not copy more data, than available */
+ if (is_kmalloc && param_size > buff_len)
+ param_size = buff_len;
+
+ if (is_kmalloc)
+ memcpy(param_read_buf, &desc_buf[param_offset], param_size);
+out:
+ if (is_kmalloc)
+ kfree(desc_buf);
+ return ret;
+}
+
+/* replace non-printable or non-ASCII characters with spaces */
+static inline void ufshcd_remove_non_printable(uint8_t *val)
+{
+ if (!val)
+ return;
+
+ if (*val < 0x20 || *val > 0x7e)
+ *val = ' ';
+}
+
+/**
+ * ufshcd_uic_pwr_ctrl - executes UIC commands (which affects the link power
+ * state) and waits for it to take effect.
+ *
+ */
+static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd)
+{
+ unsigned long start = 0;
+ u8 status;
+ int ret;
+
+ ret = ufshcd_send_uic_cmd(hba, cmd);
+ if (ret) {
+ dev_err(hba->dev,
+ "pwr ctrl cmd 0x%x with mode 0x%x uic error %d\n",
+ cmd->command, cmd->argument3, ret);
+
+ return ret;
+ }
+
+ start = get_timer(0);
+ do {
+ status = ufshcd_get_upmcrs(hba);
+ if (get_timer(start) > UFS_UIC_CMD_TIMEOUT) {
+ dev_err(hba->dev,
+ "pwr ctrl cmd 0x%x failed, host upmcrs:0x%x\n",
+ cmd->command, status);
+ ret = (status != PWR_OK) ? status : -1;
+ break;
+ }
+ } while (status != PWR_LOCAL);
+
+ return ret;
+}
+
+/**
+ * ufshcd_uic_change_pwr_mode - Perform the UIC power mode change
+ * using DME_SET primitives.
+ */
+static int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode)
+{
+ struct uic_command uic_cmd = {0};
+ int ret;
+
+ uic_cmd.command = UIC_CMD_DME_SET;
+ uic_cmd.argument1 = UIC_ARG_MIB(PA_PWRMODE);
+ uic_cmd.argument3 = mode;
+ ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
+
+ return ret;
+}
+
+static
+void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufs_hba *hba,
+ struct scsi_cmd *pccb, u32 upiu_flags)
+{
+ struct utp_upiu_req *ucd_req_ptr = hba->ucd_req_ptr;
+ unsigned int cdb_len;
+
+ /* command descriptor fields */
+ ucd_req_ptr->header.dword_0 =
+ UPIU_HEADER_DWORD(UPIU_TRANSACTION_COMMAND, upiu_flags,
+ pccb->lun, TASK_TAG);
+ ucd_req_ptr->header.dword_1 =
+ UPIU_HEADER_DWORD(UPIU_COMMAND_SET_TYPE_SCSI, 0, 0, 0);
+
+ /* Total EHS length and Data segment length will be zero */
+ ucd_req_ptr->header.dword_2 = 0;
+
+ ucd_req_ptr->sc.exp_data_transfer_len = cpu_to_be32(pccb->datalen);
+
+ cdb_len = min_t(unsigned short, pccb->cmdlen, UFS_CDB_SIZE);
+ memset(ucd_req_ptr->sc.cdb, 0, UFS_CDB_SIZE);
+ memcpy(ucd_req_ptr->sc.cdb, pccb->cmd, cdb_len);
+
+ memset(hba->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
+}
+
+static inline void prepare_prdt_desc(struct ufshcd_sg_entry *entry,
+ unsigned char *buf, ulong len)
+{
+ entry->size = cpu_to_le32(len) | GENMASK(1, 0);
+ entry->base_addr = cpu_to_le32(lower_32_bits((unsigned long)buf));
+ entry->upper_addr = cpu_to_le32(upper_32_bits((unsigned long)buf));
+}
+
+static void prepare_prdt_table(struct ufs_hba *hba, struct scsi_cmd *pccb)
+{
+ struct utp_transfer_req_desc *req_desc = hba->utrdl;
+ struct ufshcd_sg_entry *prd_table = hba->ucd_prdt_ptr;
+ ulong datalen = pccb->datalen;
+ int table_length;
+ u8 *buf;
+ int i;
+
+ if (!datalen) {
+ req_desc->prd_table_length = 0;
+ return;
+ }
+
+ table_length = DIV_ROUND_UP(pccb->datalen, MAX_PRDT_ENTRY);
+ buf = pccb->pdata;
+ i = table_length;
+ while (--i) {
+ prepare_prdt_desc(&prd_table[table_length - i - 1], buf,
+ MAX_PRDT_ENTRY - 1);
+ buf += MAX_PRDT_ENTRY;
+ datalen -= MAX_PRDT_ENTRY;
+ }
+
+ prepare_prdt_desc(&prd_table[table_length - i - 1], buf, datalen - 1);
+
+ req_desc->prd_table_length = table_length;
+}
+
+static int ufs_scsi_exec(struct udevice *scsi_dev, struct scsi_cmd *pccb)
+{
+ struct ufs_hba *hba = dev_get_uclass_priv(scsi_dev->parent);
+ struct utp_transfer_req_desc *req_desc = hba->utrdl;
+ u32 upiu_flags;
+ int ocs, result = 0;
+ u8 scsi_status;
+
+ ufshcd_prepare_req_desc_hdr(req_desc, &upiu_flags, pccb->dma_dir);
+ ufshcd_prepare_utp_scsi_cmd_upiu(hba, pccb, upiu_flags);
+ prepare_prdt_table(hba, pccb);
+
+ ufshcd_send_command(hba, TASK_TAG);
+
+ ocs = ufshcd_get_tr_ocs(hba);
+ switch (ocs) {
+ case OCS_SUCCESS:
+ result = ufshcd_get_req_rsp(hba->ucd_rsp_ptr);
+ switch (result) {
+ case UPIU_TRANSACTION_RESPONSE:
+ result = ufshcd_get_rsp_upiu_result(hba->ucd_rsp_ptr);
+
+ scsi_status = result & MASK_SCSI_STATUS;
+ if (scsi_status)
+ return -EINVAL;
+
+ break;
+ case UPIU_TRANSACTION_REJECT_UPIU:
+ /* TODO: handle Reject UPIU Response */
+ dev_err(hba->dev,
+ "Reject UPIU not fully implemented\n");
+ return -EINVAL;
+ default:
+ dev_err(hba->dev,
+ "Unexpected request response code = %x\n",
+ result);
+ return -EINVAL;
+ }
+ break;
+ default:
+ dev_err(hba->dev, "OCS error from controller = %x\n", ocs);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+static inline int ufshcd_read_desc(struct ufs_hba *hba, enum desc_idn desc_id,
+ int desc_index, u8 *buf, u32 size)
+{
+ return ufshcd_read_desc_param(hba, desc_id, desc_index, 0, buf, size);
+}
+
+static int ufshcd_read_device_desc(struct ufs_hba *hba, u8 *buf, u32 size)
+{
+ return ufshcd_read_desc(hba, QUERY_DESC_IDN_DEVICE, 0, buf, size);
+}
+
+/**
+ * ufshcd_read_string_desc - read string descriptor
+ *
+ */
+int ufshcd_read_string_desc(struct ufs_hba *hba, int desc_index,
+ u8 *buf, u32 size, bool ascii)
+{
+ int err = 0;
+
+ err = ufshcd_read_desc(hba, QUERY_DESC_IDN_STRING, desc_index, buf,
+ size);
+
+ if (err) {
+ dev_err(hba->dev, "%s: reading String Desc failed after %d retries. err = %d\n",
+ __func__, QUERY_REQ_RETRIES, err);
+ goto out;
+ }
+
+ if (ascii) {
+ int desc_len;
+ int ascii_len;
+ int i;
+ u8 *buff_ascii;
+
+ desc_len = buf[0];
+ /* remove header and divide by 2 to move from UTF16 to UTF8 */
+ ascii_len = (desc_len - QUERY_DESC_HDR_SIZE) / 2 + 1;
+ if (size < ascii_len + QUERY_DESC_HDR_SIZE) {
+ dev_err(hba->dev, "%s: buffer allocated size is too small\n",
+ __func__);
+ err = -ENOMEM;
+ goto out;
+ }
+
+ buff_ascii = kmalloc(ascii_len, GFP_KERNEL);
+ if (!buff_ascii) {
+ err = -ENOMEM;
+ goto out;
+ }
+
+ /*
+ * the descriptor contains string in UTF16 format
+ * we need to convert to utf-8 so it can be displayed
+ */
+ utf16_to_utf8(buff_ascii,
+ (uint16_t *)&buf[QUERY_DESC_HDR_SIZE], ascii_len);
+
+ /* replace non-printable or non-ASCII characters with spaces */
+ for (i = 0; i < ascii_len; i++)
+ ufshcd_remove_non_printable(&buff_ascii[i]);
+
+ memset(buf + QUERY_DESC_HDR_SIZE, 0,
+ size - QUERY_DESC_HDR_SIZE);
+ memcpy(buf + QUERY_DESC_HDR_SIZE, buff_ascii, ascii_len);
+ buf[QUERY_DESC_LENGTH_OFFSET] = ascii_len + QUERY_DESC_HDR_SIZE;
+ kfree(buff_ascii);
+ }
+out:
+ return err;
+}
+
+static int ufs_get_device_desc(struct ufs_hba *hba,
+ struct ufs_dev_desc *dev_desc)
+{
+ int err;
+ size_t buff_len;
+ u8 model_index;
+ u8 *desc_buf;
+
+ buff_len = max_t(size_t, hba->desc_size.dev_desc,
+ QUERY_DESC_MAX_SIZE + 1);
+ desc_buf = kmalloc(buff_len, GFP_KERNEL);
+ if (!desc_buf) {
+ err = -ENOMEM;
+ goto out;
+ }
+
+ err = ufshcd_read_device_desc(hba, desc_buf, hba->desc_size.dev_desc);
+ if (err) {
+ dev_err(hba->dev, "%s: Failed reading Device Desc. err = %d\n",
+ __func__, err);
+ goto out;
+ }
+
+ /*
+ * getting vendor (manufacturerID) and Bank Index in big endian
+ * format
+ */
+ dev_desc->wmanufacturerid = desc_buf[DEVICE_DESC_PARAM_MANF_ID] << 8 |
+ desc_buf[DEVICE_DESC_PARAM_MANF_ID + 1];
+
+ model_index = desc_buf[DEVICE_DESC_PARAM_PRDCT_NAME];
+
+ /* Zero-pad entire buffer for string termination. */
+ memset(desc_buf, 0, buff_len);
+
+ err = ufshcd_read_string_desc(hba, model_index, desc_buf,
+ QUERY_DESC_MAX_SIZE, true/*ASCII*/);
+ if (err) {
+ dev_err(hba->dev, "%s: Failed reading Product Name. err = %d\n",
+ __func__, err);
+ goto out;
+ }
+
+ desc_buf[QUERY_DESC_MAX_SIZE] = '\0';
+ strlcpy(dev_desc->model, (char *)(desc_buf + QUERY_DESC_HDR_SIZE),
+ min_t(u8, desc_buf[QUERY_DESC_LENGTH_OFFSET],
+ MAX_MODEL_LEN));
+
+ /* Null terminate the model string */
+ dev_desc->model[MAX_MODEL_LEN] = '\0';
+
+out:
+ kfree(desc_buf);
+ return err;
+}
+
+/**
+ * ufshcd_get_max_pwr_mode - reads the max power mode negotiated with device
+ */
+static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba)
+{
+ struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info;
+
+ if (hba->max_pwr_info.is_valid)
+ return 0;
+
+ pwr_info->pwr_tx = FAST_MODE;
+ pwr_info->pwr_rx = FAST_MODE;
+ pwr_info->hs_rate = PA_HS_MODE_B;
+
+ /* Get the connected lane count */
+ ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDRXDATALANES),
+ &pwr_info->lane_rx);
+ ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
+ &pwr_info->lane_tx);
+
+ if (!pwr_info->lane_rx || !pwr_info->lane_tx) {
+ dev_err(hba->dev, "%s: invalid connected lanes value. rx=%d, tx=%d\n",
+ __func__, pwr_info->lane_rx, pwr_info->lane_tx);
+ return -EINVAL;
+ }
+
+ /*
+ * First, get the maximum gears of HS speed.
+ * If a zero value, it means there is no HSGEAR capability.
+ * Then, get the maximum gears of PWM speed.
+ */
+ ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), &pwr_info->gear_rx);
+ if (!pwr_info->gear_rx) {
+ ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
+ &pwr_info->gear_rx);
+ if (!pwr_info->gear_rx) {
+ dev_err(hba->dev, "%s: invalid max pwm rx gear read = %d\n",
+ __func__, pwr_info->gear_rx);
+ return -EINVAL;
+ }
+ pwr_info->pwr_rx = SLOW_MODE;
+ }
+
+ ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR),
+ &pwr_info->gear_tx);
+ if (!pwr_info->gear_tx) {
+ ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
+ &pwr_info->gear_tx);
+ if (!pwr_info->gear_tx) {
+ dev_err(hba->dev, "%s: invalid max pwm tx gear read = %d\n",
+ __func__, pwr_info->gear_tx);
+ return -EINVAL;
+ }
+ pwr_info->pwr_tx = SLOW_MODE;
+ }
+
+ hba->max_pwr_info.is_valid = true;
+ return 0;
+}
+
+static int ufshcd_change_power_mode(struct ufs_hba *hba,
+ struct ufs_pa_layer_attr *pwr_mode)
+{
+ int ret;
+
+ /* if already configured to the requested pwr_mode */
+ if (pwr_mode->gear_rx == hba->pwr_info.gear_rx &&
+ pwr_mode->gear_tx == hba->pwr_info.gear_tx &&
+ pwr_mode->lane_rx == hba->pwr_info.lane_rx &&
+ pwr_mode->lane_tx == hba->pwr_info.lane_tx &&
+ pwr_mode->pwr_rx == hba->pwr_info.pwr_rx &&
+ pwr_mode->pwr_tx == hba->pwr_info.pwr_tx &&
+ pwr_mode->hs_rate == hba->pwr_info.hs_rate) {
+ dev_dbg(hba->dev, "%s: power already configured\n", __func__);
+ return 0;
+ }
+
+ /*
+ * Configure attributes for power mode change with below.
+ * - PA_RXGEAR, PA_ACTIVERXDATALANES, PA_RXTERMINATION,
+ * - PA_TXGEAR, PA_ACTIVETXDATALANES, PA_TXTERMINATION,
+ * - PA_HSSERIES
+ */
+ ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXGEAR), pwr_mode->gear_rx);
+ ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVERXDATALANES),
+ pwr_mode->lane_rx);
+ if (pwr_mode->pwr_rx == FASTAUTO_MODE || pwr_mode->pwr_rx == FAST_MODE)
+ ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), TRUE);
+ else
+ ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), FALSE);
+
+ ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXGEAR), pwr_mode->gear_tx);
+ ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVETXDATALANES),
+ pwr_mode->lane_tx);
+ if (pwr_mode->pwr_tx == FASTAUTO_MODE || pwr_mode->pwr_tx == FAST_MODE)
+ ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), TRUE);
+ else
+ ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), FALSE);
+
+ if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
+ pwr_mode->pwr_tx == FASTAUTO_MODE ||
+ pwr_mode->pwr_rx == FAST_MODE ||
+ pwr_mode->pwr_tx == FAST_MODE)
+ ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HSSERIES),
+ pwr_mode->hs_rate);
+
+ ret = ufshcd_uic_change_pwr_mode(hba, pwr_mode->pwr_rx << 4 |
+ pwr_mode->pwr_tx);
+
+ if (ret) {
+ dev_err(hba->dev,
+ "%s: power mode change failed %d\n", __func__, ret);
+
+ return ret;
+ }
+
+ /* Copy new Power Mode to power info */
+ memcpy(&hba->pwr_info, pwr_mode, sizeof(struct ufs_pa_layer_attr));
+
+ return ret;
+}
+
+/**
+ * ufshcd_verify_dev_init() - Verify device initialization
+ *
+ */
+static int ufshcd_verify_dev_init(struct ufs_hba *hba)
+{
+ int retries;
+ int err;
+
+ for (retries = NOP_OUT_RETRIES; retries > 0; retries--) {
+ err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_NOP,
+ NOP_OUT_TIMEOUT);
+ if (!err || err == -ETIMEDOUT)
+ break;
+
+ dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err);
+ }
+
+ if (err)
+ dev_err(hba->dev, "%s: NOP OUT failed %d\n", __func__, err);
+
+ return err;
+}
+
+/**
+ * ufshcd_complete_dev_init() - checks device readiness
+ */
+static int ufshcd_complete_dev_init(struct ufs_hba *hba)
+{
+ int i;
+ int err;
+ bool flag_res = 1;
+
+ err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
+ QUERY_FLAG_IDN_FDEVICEINIT, NULL);
+ if (err) {
+ dev_err(hba->dev,
+ "%s setting fDeviceInit flag failed with error %d\n",
+ __func__, err);
+ goto out;
+ }
+
+ /* poll for max. 1000 iterations for fDeviceInit flag to clear */
+ for (i = 0; i < 1000 && !err && flag_res; i++)
+ err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG,
+ QUERY_FLAG_IDN_FDEVICEINIT,
+ &flag_res);
+
+ if (err)
+ dev_err(hba->dev,
+ "%s reading fDeviceInit flag failed with error %d\n",
+ __func__, err);
+ else if (flag_res)
+ dev_err(hba->dev,
+ "%s fDeviceInit was not cleared by the device\n",
+ __func__);
+
+out:
+ return err;
+}
+
+static void ufshcd_def_desc_sizes(struct ufs_hba *hba)
+{
+ hba->desc_size.dev_desc = QUERY_DESC_DEVICE_DEF_SIZE;
+ hba->desc_size.pwr_desc = QUERY_DESC_POWER_DEF_SIZE;
+ hba->desc_size.interc_desc = QUERY_DESC_INTERCONNECT_DEF_SIZE;
+ hba->desc_size.conf_desc = QUERY_DESC_CONFIGURATION_DEF_SIZE;
+ hba->desc_size.unit_desc = QUERY_DESC_UNIT_DEF_SIZE;
+ hba->desc_size.geom_desc = QUERY_DESC_GEOMETRY_DEF_SIZE;
+ hba->desc_size.hlth_desc = QUERY_DESC_HEALTH_DEF_SIZE;
+}
+
+int ufs_start(struct ufs_hba *hba)
+{
+ struct ufs_dev_desc card = {0};
+ int ret;
+
+ ret = ufshcd_link_startup(hba);
+ if (ret)
+ return ret;
+
+ ret = ufshcd_verify_dev_init(hba);
+ if (ret)
+ return ret;
+
+ ret = ufshcd_complete_dev_init(hba);
+ if (ret)
+ return ret;
+
+ /* Init check for device descriptor sizes */
+ ufshcd_init_desc_sizes(hba);
+
+ ret = ufs_get_device_desc(hba, &card);
+ if (ret) {
+ dev_err(hba->dev, "%s: Failed getting device info. err = %d\n",
+ __func__, ret);
+
+ return ret;
+ }
+
+ if (ufshcd_get_max_pwr_mode(hba)) {
+ dev_err(hba->dev,
+ "%s: Failed getting max supported power mode\n",
+ __func__);
+ } else {
+ ret = ufshcd_change_power_mode(hba, &hba->max_pwr_info.info);
+ if (ret) {
+ dev_err(hba->dev, "%s: Failed setting power mode, err = %d\n",
+ __func__, ret);
+
+ return ret;
+ }
+
+ printf("Device at %s up at:", hba->dev->name);
+ ufshcd_print_pwr_info(hba);
+ }
+
+ return 0;
+}
+
+int ufshcd_probe(struct udevice *ufs_dev, struct ufs_hba_ops *hba_ops)
+{
+ struct ufs_hba *hba = dev_get_uclass_priv(ufs_dev);
+ struct scsi_platdata *scsi_plat;
+ struct udevice *scsi_dev;
+ int err;
+
+ device_find_first_child(ufs_dev, &scsi_dev);
+ if (!scsi_dev)
+ return -ENODEV;
+
+ scsi_plat = dev_get_uclass_platdata(scsi_dev);
+ scsi_plat->max_id = UFSHCD_MAX_ID;
+ scsi_plat->max_lun = UFS_MAX_LUNS;
+ scsi_plat->max_bytes_per_req = UFS_MAX_BYTES;
+
+ hba->dev = ufs_dev;
+ hba->ops = hba_ops;
+ hba->mmio_base = (void *)dev_read_addr(ufs_dev);
+
+ /* Set descriptor lengths to specification defaults */
+ ufshcd_def_desc_sizes(hba);
+
+ ufshcd_ops_init(hba);
+
+ /* Read capabilties registers */
+ hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES);
+
+ /* Get UFS version supported by the controller */
+ hba->version = ufshcd_get_ufs_version(hba);
+ if (hba->version != UFSHCI_VERSION_10 &&
+ hba->version != UFSHCI_VERSION_11 &&
+ hba->version != UFSHCI_VERSION_20 &&
+ hba->version != UFSHCI_VERSION_21)
+ dev_err(hba->dev, "invalid UFS version 0x%x\n",
+ hba->version);
+
+ /* Get Interrupt bit mask per version */
+ hba->intr_mask = ufshcd_get_intr_mask(hba);
+
+ /* Allocate memory for host memory space */
+ err = ufshcd_memory_alloc(hba);
+ if (err) {
+ dev_err(hba->dev, "Memory allocation failed\n");
+ return err;
+ }
+
+ /* Configure Local data structures */
+ ufshcd_host_memory_configure(hba);
+
+ /*
+ * In order to avoid any spurious interrupt immediately after
+ * registering UFS controller interrupt handler, clear any pending UFS
+ * interrupt status and disable all the UFS interrupts.
+ */
+ ufshcd_writel(hba, ufshcd_readl(hba, REG_INTERRUPT_STATUS),
+ REG_INTERRUPT_STATUS);
+ ufshcd_writel(hba, 0, REG_INTERRUPT_ENABLE);
+
+ err = ufshcd_hba_enable(hba);
+ if (err) {
+ dev_err(hba->dev, "Host controller enable failed\n");
+ return err;
+ }
+
+ err = ufs_start(hba);
+ if (err)
+ return err;
+
+ return 0;
+}
+
+int ufs_scsi_bind(struct udevice *ufs_dev, struct udevice **scsi_devp)
+{
+ int ret = device_bind_driver(ufs_dev, "ufs_scsi", "ufs_scsi",
+ scsi_devp);
+
+ return ret;
+}
+
+static struct scsi_ops ufs_ops = {
+ .exec = ufs_scsi_exec,
+};
+
+int ufs_probe_dev(int index)
+{
+ struct udevice *dev;
+
+ return uclass_get_device(UCLASS_UFS, index, &dev);
+}
+
+int ufs_probe(void)
+{
+ struct udevice *dev;
+ int ret, i;
+
+ for (i = 0;; i++) {
+ ret = uclass_get_device(UCLASS_UFS, i, &dev);
+ if (ret == -ENODEV)
+ break;
+ }
+
+ return 0;
+}
+
+U_BOOT_DRIVER(ufs_scsi) = {
+ .id = UCLASS_SCSI,
+ .name = "ufs_scsi",
+ .ops = &ufs_ops,
+};
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Cadence USBSS DRD Driver - gadget side.
+ *
+ * Copyright (C) 2018-2019 Cadence Design Systems.
+ * Copyright (C) 2017-2018 NXP
+ *
+ * Authors: Pawel Jez <pjez@cadence.com>,
+ * Pawel Laszczak <pawell@cadence.com>
+ * Peter Chen <peter.chen@nxp.com>
+ */
+
+/*
+ * Work around 1:
+ * At some situations, the controller may get stale data address in TRB
+ * at below sequences:
+ * 1. Controller read TRB includes data address
+ * 2. Software updates TRBs includes data address and Cycle bit
+ * 3. Controller read TRB which includes Cycle bit
+ * 4. DMA run with stale data address
+ *
+ * To fix this problem, driver needs to make the first TRB in TD as invalid.
+ * After preparing all TRBs driver needs to check the position of DMA and
+ * if the DMA point to the first just added TRB and doorbell is 1,
+ * then driver must defer making this TRB as valid. This TRB will be make
+ * as valid during adding next TRB only if DMA is stopped or at TRBERR
+ * interrupt.
+ *
+ * Issue has been fixed in DEV_VER_V3 version of controller.
+ *
+ * Work around 2:
+ * Controller for OUT endpoints has shared on-chip buffers for all incoming
+ * packets, including ep0out. It's FIFO buffer, so packets must be handle by DMA
+ * in correct order. If the first packet in the buffer will not be handled,
+ * then the following packets directed for other endpoints and functions
+ * will be blocked.
+ * Additionally the packets directed to one endpoint can block entire on-chip
+ * buffers. In this case transfer to other endpoints also will blocked.
+ *
+ * To resolve this issue after raising the descriptor missing interrupt
+ * driver prepares internal usb_request object and use it to arm DMA transfer.
+ *
+ * The problematic situation was observed in case when endpoint has been enabled
+ * but no usb_request were queued. Driver try detects such endpoints and will
+ * use this workaround only for these endpoint.
+ *
+ * Driver use limited number of buffer. This number can be set by macro
+ * CDNS3_WA2_NUM_BUFFERS.
+ *
+ * Such blocking situation was observed on ACM gadget. For this function
+ * host send OUT data packet but ACM function is not prepared for this packet.
+ * It's cause that buffer placed in on chip memory block transfer to other
+ * endpoints.
+ *
+ * Issue has been fixed in DEV_VER_V2 version of controller.
+ *
+ */
+
+#include <dm.h>
+#include <dm/device_compat.h>
+#include <dm/devres.h>
+#include <linux/err.h>
+#include <linux/usb/gadget.h>
+#include <linux/compat.h>
+#include <linux/iopoll.h>
+#include <linux/dma-mapping.h>
+#include <linux/bitmap.h>
+#include <linux/bug.h>
+
+#include "core.h"
+#include "gadget-export.h"
+#include "gadget.h"
+#include "trace.h"
+#include "drd.h"
+
+#define readl_poll_timeout_atomic readl_poll_timeout
+#define usleep_range(a, b) udelay((b))
+
+static int __cdns3_gadget_ep_queue(struct usb_ep *ep,
+ struct usb_request *request,
+ gfp_t gfp_flags);
+
+/**
+ * cdns3_set_register_bit - set bit in given register.
+ * @ptr: address of device controller register to be read and changed
+ * @mask: bits requested to set
+ */
+void cdns3_set_register_bit(void __iomem *ptr, u32 mask)
+{
+ mask = readl(ptr) | mask;
+ writel(mask, ptr);
+}
+
+/**
+ * cdns3_ep_addr_to_index - Macro converts endpoint address to
+ * index of endpoint object in cdns3_device.eps[] container
+ * @ep_addr: endpoint address for which endpoint object is required
+ *
+ */
+u8 cdns3_ep_addr_to_index(u8 ep_addr)
+{
+ return (((ep_addr & 0x7F)) + ((ep_addr & USB_DIR_IN) ? 16 : 0));
+}
+
+static int cdns3_get_dma_pos(struct cdns3_device *priv_dev,
+ struct cdns3_endpoint *priv_ep)
+{
+ int dma_index;
+
+ dma_index = readl(&priv_dev->regs->ep_traddr) - priv_ep->trb_pool_dma;
+
+ return dma_index / TRB_SIZE;
+}
+
+/**
+ * cdns3_next_request - returns next request from list
+ * @list: list containing requests
+ *
+ * Returns request or NULL if no requests in list
+ */
+struct usb_request *cdns3_next_request(struct list_head *list)
+{
+ return list_first_entry_or_null(list, struct usb_request, list);
+}
+
+/**
+ * cdns3_next_align_buf - returns next buffer from list
+ * @list: list containing buffers
+ *
+ * Returns buffer or NULL if no buffers in list
+ */
+struct cdns3_aligned_buf *cdns3_next_align_buf(struct list_head *list)
+{
+ return list_first_entry_or_null(list, struct cdns3_aligned_buf, list);
+}
+
+/**
+ * cdns3_next_priv_request - returns next request from list
+ * @list: list containing requests
+ *
+ * Returns request or NULL if no requests in list
+ */
+struct cdns3_request *cdns3_next_priv_request(struct list_head *list)
+{
+ return list_first_entry_or_null(list, struct cdns3_request, list);
+}
+
+/**
+ * select_ep - selects endpoint
+ * @priv_dev: extended gadget object
+ * @ep: endpoint address
+ */
+void cdns3_select_ep(struct cdns3_device *priv_dev, u32 ep)
+{
+ if (priv_dev->selected_ep == ep)
+ return;
+
+ priv_dev->selected_ep = ep;
+ writel(ep, &priv_dev->regs->ep_sel);
+}
+
+dma_addr_t cdns3_trb_virt_to_dma(struct cdns3_endpoint *priv_ep,
+ struct cdns3_trb *trb)
+{
+ u32 offset = (char *)trb - (char *)priv_ep->trb_pool;
+
+ return priv_ep->trb_pool_dma + offset;
+}
+
+int cdns3_ring_size(struct cdns3_endpoint *priv_ep)
+{
+ switch (priv_ep->type) {
+ case USB_ENDPOINT_XFER_ISOC:
+ return TRB_ISO_RING_SIZE;
+ case USB_ENDPOINT_XFER_CONTROL:
+ return TRB_CTRL_RING_SIZE;
+ default:
+ return TRB_RING_SIZE;
+ }
+}
+
+/**
+ * cdns3_allocate_trb_pool - Allocates TRB's pool for selected endpoint
+ * @priv_ep: endpoint object
+ *
+ * Function will return 0 on success or -ENOMEM on allocation error
+ */
+int cdns3_allocate_trb_pool(struct cdns3_endpoint *priv_ep)
+{
+ int ring_size = cdns3_ring_size(priv_ep);
+ struct cdns3_trb *link_trb;
+
+ if (!priv_ep->trb_pool) {
+ priv_ep->trb_pool =
+ dma_alloc_coherent(ring_size,
+ (unsigned long *)&priv_ep->trb_pool_dma);
+ if (!priv_ep->trb_pool)
+ return -ENOMEM;
+ } else {
+ memset(priv_ep->trb_pool, 0, ring_size);
+ }
+
+ if (!priv_ep->num)
+ return 0;
+
+ priv_ep->num_trbs = ring_size / TRB_SIZE;
+ /* Initialize the last TRB as Link TRB. */
+ link_trb = (priv_ep->trb_pool + (priv_ep->num_trbs - 1));
+ link_trb->buffer = TRB_BUFFER(priv_ep->trb_pool_dma);
+ link_trb->control = TRB_CYCLE | TRB_TYPE(TRB_LINK) | TRB_TOGGLE;
+
+ return 0;
+}
+
+static void cdns3_free_trb_pool(struct cdns3_endpoint *priv_ep)
+{
+ if (priv_ep->trb_pool) {
+ dma_free_coherent(priv_ep->trb_pool);
+ priv_ep->trb_pool = NULL;
+ }
+}
+
+/**
+ * cdns3_ep_stall_flush - Stalls and flushes selected endpoint
+ * @priv_ep: endpoint object
+ *
+ * Endpoint must be selected before call to this function
+ */
+static void cdns3_ep_stall_flush(struct cdns3_endpoint *priv_ep)
+{
+ struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+ int val;
+
+ trace_cdns3_halt(priv_ep, 1, 1);
+
+ writel(EP_CMD_DFLUSH | EP_CMD_ERDY | EP_CMD_SSTALL,
+ &priv_dev->regs->ep_cmd);
+
+ /* wait for DFLUSH cleared */
+ readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
+ !(val & EP_CMD_DFLUSH), 1000);
+ priv_ep->flags |= EP_STALLED;
+ priv_ep->flags &= ~EP_STALL_PENDING;
+}
+
+/**
+ * cdns3_hw_reset_eps_config - reset endpoints configuration kept by controller.
+ * @priv_dev: extended gadget object
+ */
+void cdns3_hw_reset_eps_config(struct cdns3_device *priv_dev)
+{
+ writel(USB_CONF_CFGRST, &priv_dev->regs->usb_conf);
+
+ cdns3_allow_enable_l1(priv_dev, 0);
+ priv_dev->hw_configured_flag = 0;
+ priv_dev->onchip_used_size = 0;
+ priv_dev->out_mem_is_allocated = 0;
+ priv_dev->wait_for_setup = 0;
+}
+
+/**
+ * cdns3_ep_inc_trb - increment a trb index.
+ * @index: Pointer to the TRB index to increment.
+ * @cs: Cycle state
+ * @trb_in_seg: number of TRBs in segment
+ *
+ * The index should never point to the link TRB. After incrementing,
+ * if it is point to the link TRB, wrap around to the beginning and revert
+ * cycle state bit The
+ * link TRB is always at the last TRB entry.
+ */
+static void cdns3_ep_inc_trb(int *index, u8 *cs, int trb_in_seg)
+{
+ (*index)++;
+ if (*index == (trb_in_seg - 1)) {
+ *index = 0;
+ *cs ^= 1;
+ }
+}
+
+/**
+ * cdns3_ep_inc_enq - increment endpoint's enqueue pointer
+ * @priv_ep: The endpoint whose enqueue pointer we're incrementing
+ */
+static void cdns3_ep_inc_enq(struct cdns3_endpoint *priv_ep)
+{
+ priv_ep->free_trbs--;
+ cdns3_ep_inc_trb(&priv_ep->enqueue, &priv_ep->pcs, priv_ep->num_trbs);
+}
+
+/**
+ * cdns3_ep_inc_deq - increment endpoint's dequeue pointer
+ * @priv_ep: The endpoint whose dequeue pointer we're incrementing
+ */
+static void cdns3_ep_inc_deq(struct cdns3_endpoint *priv_ep)
+{
+ priv_ep->free_trbs++;
+ cdns3_ep_inc_trb(&priv_ep->dequeue, &priv_ep->ccs, priv_ep->num_trbs);
+}
+
+void cdns3_move_deq_to_next_trb(struct cdns3_request *priv_req)
+{
+ struct cdns3_endpoint *priv_ep = priv_req->priv_ep;
+ int current_trb = priv_req->start_trb;
+
+ while (current_trb != priv_req->end_trb) {
+ cdns3_ep_inc_deq(priv_ep);
+ current_trb = priv_ep->dequeue;
+ }
+
+ cdns3_ep_inc_deq(priv_ep);
+}
+
+/**
+ * cdns3_allow_enable_l1 - enable/disable permits to transition to L1.
+ * @priv_dev: Extended gadget object
+ * @enable: Enable/disable permit to transition to L1.
+ *
+ * If bit USB_CONF_L1EN is set and device receive Extended Token packet,
+ * then controller answer with ACK handshake.
+ * If bit USB_CONF_L1DS is set and device receive Extended Token packet,
+ * then controller answer with NYET handshake.
+ */
+void cdns3_allow_enable_l1(struct cdns3_device *priv_dev, int enable)
+{
+ if (enable)
+ writel(USB_CONF_L1EN, &priv_dev->regs->usb_conf);
+ else
+ writel(USB_CONF_L1DS, &priv_dev->regs->usb_conf);
+}
+
+enum usb_device_speed cdns3_get_speed(struct cdns3_device *priv_dev)
+{
+ u32 reg;
+
+ reg = readl(&priv_dev->regs->usb_sts);
+
+ if (DEV_SUPERSPEED(reg))
+ return USB_SPEED_SUPER;
+ else if (DEV_HIGHSPEED(reg))
+ return USB_SPEED_HIGH;
+ else if (DEV_FULLSPEED(reg))
+ return USB_SPEED_FULL;
+ else if (DEV_LOWSPEED(reg))
+ return USB_SPEED_LOW;
+ return USB_SPEED_UNKNOWN;
+}
+
+/**
+ * cdns3_start_all_request - add to ring all request not started
+ * @priv_dev: Extended gadget object
+ * @priv_ep: The endpoint for whom request will be started.
+ *
+ * Returns return ENOMEM if transfer ring i not enough TRBs to start
+ * all requests.
+ */
+static int cdns3_start_all_request(struct cdns3_device *priv_dev,
+ struct cdns3_endpoint *priv_ep)
+{
+ struct usb_request *request;
+ int ret = 0;
+
+ while (!list_empty(&priv_ep->deferred_req_list)) {
+ request = cdns3_next_request(&priv_ep->deferred_req_list);
+
+ ret = cdns3_ep_run_transfer(priv_ep, request);
+ if (ret)
+ return ret;
+
+ list_del(&request->list);
+ list_add_tail(&request->list,
+ &priv_ep->pending_req_list);
+ }
+
+ priv_ep->flags &= ~EP_RING_FULL;
+ return ret;
+}
+
+/*
+ * WA2: Set flag for all not ISOC OUT endpoints. If this flag is set
+ * driver try to detect whether endpoint need additional internal
+ * buffer for unblocking on-chip FIFO buffer. This flag will be cleared
+ * if before first DESCMISS interrupt the DMA will be armed.
+ */
+#define cdns3_wa2_enable_detection(priv_dev, ep_priv, reg) do { \
+ if (!priv_ep->dir && priv_ep->type != USB_ENDPOINT_XFER_ISOC) { \
+ priv_ep->flags |= EP_QUIRK_EXTRA_BUF_DET; \
+ (reg) |= EP_STS_EN_DESCMISEN; \
+ } } while (0)
+
+/**
+ * cdns3_wa2_descmiss_copy_data copy data from internal requests to
+ * request queued by class driver.
+ * @priv_ep: extended endpoint object
+ * @request: request object
+ */
+static void cdns3_wa2_descmiss_copy_data(struct cdns3_endpoint *priv_ep,
+ struct usb_request *request)
+{
+ struct usb_request *descmiss_req;
+ struct cdns3_request *descmiss_priv_req;
+
+ while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
+ int chunk_end;
+ int length;
+
+ descmiss_priv_req =
+ cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
+ descmiss_req = &descmiss_priv_req->request;
+
+ /* driver can't touch pending request */
+ if (descmiss_priv_req->flags & REQUEST_PENDING)
+ break;
+
+ chunk_end = descmiss_priv_req->flags & REQUEST_INTERNAL_CH;
+ length = request->actual + descmiss_req->actual;
+
+ request->status = descmiss_req->status;
+
+ if (length <= request->length) {
+ memcpy(&((u8 *)request->buf)[request->actual],
+ descmiss_req->buf,
+ descmiss_req->actual);
+ request->actual = length;
+ } else {
+ /* It should never occur */
+ request->status = -ENOMEM;
+ }
+
+ list_del_init(&descmiss_priv_req->list);
+
+ kfree(descmiss_req->buf);
+ cdns3_gadget_ep_free_request(&priv_ep->endpoint, descmiss_req);
+ --priv_ep->wa2_counter;
+
+ if (!chunk_end)
+ break;
+ }
+}
+
+struct usb_request *cdns3_wa2_gadget_giveback(struct cdns3_device *priv_dev,
+ struct cdns3_endpoint *priv_ep,
+ struct cdns3_request *priv_req)
+{
+ if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN &&
+ priv_req->flags & REQUEST_INTERNAL) {
+ struct usb_request *req;
+
+ req = cdns3_next_request(&priv_ep->deferred_req_list);
+
+ priv_ep->descmis_req = NULL;
+
+ if (!req)
+ return NULL;
+
+ cdns3_wa2_descmiss_copy_data(priv_ep, req);
+ if (!(priv_ep->flags & EP_QUIRK_END_TRANSFER) &&
+ req->length != req->actual) {
+ /* wait for next part of transfer */
+ return NULL;
+ }
+
+ if (req->status == -EINPROGRESS)
+ req->status = 0;
+
+ list_del_init(&req->list);
+ cdns3_start_all_request(priv_dev, priv_ep);
+ return req;
+ }
+
+ return &priv_req->request;
+}
+
+int cdns3_wa2_gadget_ep_queue(struct cdns3_device *priv_dev,
+ struct cdns3_endpoint *priv_ep,
+ struct cdns3_request *priv_req)
+{
+ int deferred = 0;
+
+ /*
+ * If transfer was queued before DESCMISS appear than we
+ * can disable handling of DESCMISS interrupt. Driver assumes that it
+ * can disable special treatment for this endpoint.
+ */
+ if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) {
+ u32 reg;
+
+ cdns3_select_ep(priv_dev, priv_ep->num | priv_ep->dir);
+ priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET;
+ reg = readl(&priv_dev->regs->ep_sts_en);
+ reg &= ~EP_STS_EN_DESCMISEN;
+ trace_cdns3_wa2(priv_ep, "workaround disabled\n");
+ writel(reg, &priv_dev->regs->ep_sts_en);
+ }
+
+ if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) {
+ u8 pending_empty = list_empty(&priv_ep->pending_req_list);
+ u8 descmiss_empty = list_empty(&priv_ep->wa2_descmiss_req_list);
+
+ /*
+ * DESCMISS transfer has been finished, so data will be
+ * directly copied from internal allocated usb_request
+ * objects.
+ */
+ if (pending_empty && !descmiss_empty &&
+ !(priv_req->flags & REQUEST_INTERNAL)) {
+ cdns3_wa2_descmiss_copy_data(priv_ep,
+ &priv_req->request);
+
+ trace_cdns3_wa2(priv_ep, "get internal stored data");
+
+ list_add_tail(&priv_req->request.list,
+ &priv_ep->pending_req_list);
+ cdns3_gadget_giveback(priv_ep, priv_req,
+ priv_req->request.status);
+
+ /*
+ * Intentionally driver returns positive value as
+ * correct value. It informs that transfer has
+ * been finished.
+ */
+ return EINPROGRESS;
+ }
+
+ /*
+ * Driver will wait for completion DESCMISS transfer,
+ * before starts new, not DESCMISS transfer.
+ */
+ if (!pending_empty && !descmiss_empty) {
+ trace_cdns3_wa2(priv_ep, "wait for pending transfer\n");
+ deferred = 1;
+ }
+
+ if (priv_req->flags & REQUEST_INTERNAL)
+ list_add_tail(&priv_req->list,
+ &priv_ep->wa2_descmiss_req_list);
+ }
+
+ return deferred;
+}
+
+static void cdns3_wa2_remove_old_request(struct cdns3_endpoint *priv_ep)
+{
+ struct cdns3_request *priv_req;
+
+ while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
+ u8 chain;
+
+ priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
+ chain = !!(priv_req->flags & REQUEST_INTERNAL_CH);
+
+ trace_cdns3_wa2(priv_ep, "removes eldest request");
+
+ kfree(priv_req->request.buf);
+ cdns3_gadget_ep_free_request(&priv_ep->endpoint,
+ &priv_req->request);
+ list_del_init(&priv_req->list);
+ --priv_ep->wa2_counter;
+
+ if (!chain)
+ break;
+ }
+}
+
+/**
+ * cdns3_wa2_descmissing_packet - handles descriptor missing event.
+ * @priv_dev: extended gadget object
+ *
+ * This function is used only for WA2. For more information see Work around 2
+ * description.
+ */
+static void cdns3_wa2_descmissing_packet(struct cdns3_endpoint *priv_ep)
+{
+ struct cdns3_request *priv_req;
+ struct usb_request *request;
+
+ if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) {
+ priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET;
+ priv_ep->flags |= EP_QUIRK_EXTRA_BUF_EN;
+ }
+
+ trace_cdns3_wa2(priv_ep, "Description Missing detected\n");
+
+ if (priv_ep->wa2_counter >= CDNS3_WA2_NUM_BUFFERS)
+ cdns3_wa2_remove_old_request(priv_ep);
+
+ request = cdns3_gadget_ep_alloc_request(&priv_ep->endpoint,
+ GFP_ATOMIC);
+ if (!request)
+ goto err;
+
+ priv_req = to_cdns3_request(request);
+ priv_req->flags |= REQUEST_INTERNAL;
+
+ /* if this field is still assigned it indicate that transfer related
+ * with this request has not been finished yet. Driver in this
+ * case simply allocate next request and assign flag REQUEST_INTERNAL_CH
+ * flag to previous one. It will indicate that current request is
+ * part of the previous one.
+ */
+ if (priv_ep->descmis_req)
+ priv_ep->descmis_req->flags |= REQUEST_INTERNAL_CH;
+
+ priv_req->request.buf = kzalloc(CDNS3_DESCMIS_BUF_SIZE,
+ GFP_ATOMIC);
+ priv_ep->wa2_counter++;
+
+ if (!priv_req->request.buf) {
+ cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
+ goto err;
+ }
+
+ priv_req->request.length = CDNS3_DESCMIS_BUF_SIZE;
+ priv_ep->descmis_req = priv_req;
+
+ __cdns3_gadget_ep_queue(&priv_ep->endpoint,
+ &priv_ep->descmis_req->request,
+ GFP_ATOMIC);
+
+ return;
+
+err:
+ dev_err(priv_ep->cdns3_dev->dev,
+ "Failed: No sufficient memory for DESCMIS\n");
+}
+
+/**
+ * cdns3_gadget_giveback - call struct usb_request's ->complete callback
+ * @priv_ep: The endpoint to whom the request belongs to
+ * @priv_req: The request we're giving back
+ * @status: completion code for the request
+ *
+ * Must be called with controller's lock held and interrupts disabled. This
+ * function will unmap @req and call its ->complete() callback to notify upper
+ * layers that it has completed.
+ */
+void cdns3_gadget_giveback(struct cdns3_endpoint *priv_ep,
+ struct cdns3_request *priv_req,
+ int status)
+{
+ struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+ struct usb_request *request = &priv_req->request;
+
+ list_del_init(&request->list);
+
+ if (request->status == -EINPROGRESS)
+ request->status = status;
+
+ usb_gadget_unmap_request(&priv_dev->gadget, request,
+ priv_ep->dir);
+
+ if ((priv_req->flags & REQUEST_UNALIGNED) &&
+ priv_ep->dir == USB_DIR_OUT && !request->status)
+ memcpy(request->buf, priv_req->aligned_buf->buf,
+ request->length);
+
+ priv_req->flags &= ~(REQUEST_PENDING | REQUEST_UNALIGNED);
+ trace_cdns3_gadget_giveback(priv_req);
+
+ if (priv_dev->dev_ver < DEV_VER_V2) {
+ request = cdns3_wa2_gadget_giveback(priv_dev, priv_ep,
+ priv_req);
+ if (!request)
+ return;
+ }
+
+ if (request->complete) {
+ spin_unlock(&priv_dev->lock);
+ usb_gadget_giveback_request(&priv_ep->endpoint,
+ request);
+ spin_lock(&priv_dev->lock);
+ }
+
+ if (request->buf == priv_dev->zlp_buf)
+ cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
+}
+
+void cdns3_wa1_restore_cycle_bit(struct cdns3_endpoint *priv_ep)
+{
+ /* Work around for stale data address in TRB*/
+ if (priv_ep->wa1_set) {
+ trace_cdns3_wa1(priv_ep, "restore cycle bit");
+
+ priv_ep->wa1_set = 0;
+ priv_ep->wa1_trb_index = 0xFFFF;
+ if (priv_ep->wa1_cycle_bit) {
+ priv_ep->wa1_trb->control =
+ priv_ep->wa1_trb->control | 0x1;
+ } else {
+ priv_ep->wa1_trb->control =
+ priv_ep->wa1_trb->control & ~0x1;
+ }
+ }
+}
+
+static void cdns3_free_aligned_request_buf(struct cdns3_device *priv_dev)
+{
+ struct cdns3_aligned_buf *buf, *tmp;
+ unsigned long flags;
+
+ spin_lock_irqsave(&priv_dev->lock, flags);
+
+ list_for_each_entry_safe(buf, tmp, &priv_dev->aligned_buf_list, list) {
+ if (!buf->in_use) {
+ list_del(&buf->list);
+
+ /*
+ * Re-enable interrupts to free DMA capable memory.
+ * Driver can't free this memory with disabled
+ * interrupts.
+ */
+ spin_unlock_irqrestore(&priv_dev->lock, flags);
+ dma_free_coherent(buf->buf);
+ kfree(buf);
+ spin_lock_irqsave(&priv_dev->lock, flags);
+ }
+ }
+
+ spin_unlock_irqrestore(&priv_dev->lock, flags);
+}
+
+static int cdns3_prepare_aligned_request_buf(struct cdns3_request *priv_req)
+{
+ struct cdns3_endpoint *priv_ep = priv_req->priv_ep;
+ struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+ struct cdns3_aligned_buf *buf;
+
+ /* check if buffer is aligned to 8. */
+ if (!((uintptr_t)priv_req->request.buf & 0x7))
+ return 0;
+
+ buf = priv_req->aligned_buf;
+
+ if (!buf || priv_req->request.length > buf->size) {
+ buf = kzalloc(sizeof(*buf), GFP_ATOMIC);
+ if (!buf)
+ return -ENOMEM;
+
+ buf->size = priv_req->request.length;
+
+ buf->buf = dma_alloc_coherent(buf->size,
+ (unsigned long *)&buf->dma);
+ if (!buf->buf) {
+ kfree(buf);
+ return -ENOMEM;
+ }
+
+ if (priv_req->aligned_buf) {
+ trace_cdns3_free_aligned_request(priv_req);
+ priv_req->aligned_buf->in_use = 0;
+#ifndef __UBOOT__
+ queue_work(system_freezable_wq,
+ &priv_dev->aligned_buf_wq);
+#else
+ cdns3_free_aligned_request_buf(priv_dev);
+#endif
+ }
+
+ buf->in_use = 1;
+ priv_req->aligned_buf = buf;
+
+ list_add_tail(&buf->list,
+ &priv_dev->aligned_buf_list);
+ }
+
+ if (priv_ep->dir == USB_DIR_IN) {
+ memcpy(buf->buf, priv_req->request.buf,
+ priv_req->request.length);
+ }
+
+ priv_req->flags |= REQUEST_UNALIGNED;
+ trace_cdns3_prepare_aligned_request(priv_req);
+
+ return 0;
+}
+
+static int cdns3_wa1_update_guard(struct cdns3_endpoint *priv_ep,
+ struct cdns3_trb *trb)
+{
+ struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+
+ if (!priv_ep->wa1_set) {
+ u32 doorbell;
+
+ doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
+
+ if (doorbell) {
+ priv_ep->wa1_cycle_bit = priv_ep->pcs ? TRB_CYCLE : 0;
+ priv_ep->wa1_set = 1;
+ priv_ep->wa1_trb = trb;
+ priv_ep->wa1_trb_index = priv_ep->enqueue;
+ trace_cdns3_wa1(priv_ep, "set guard");
+ return 0;
+ }
+ }
+ return 1;
+}
+
+static void cdns3_wa1_tray_restore_cycle_bit(struct cdns3_device *priv_dev,
+ struct cdns3_endpoint *priv_ep)
+{
+ int dma_index;
+ u32 doorbell;
+
+ doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
+ dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
+
+ if (!doorbell || dma_index != priv_ep->wa1_trb_index)
+ cdns3_wa1_restore_cycle_bit(priv_ep);
+}
+
+/**
+ * cdns3_ep_run_transfer - start transfer on no-default endpoint hardware
+ * @priv_ep: endpoint object
+ *
+ * Returns zero on success or negative value on failure
+ */
+int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
+ struct usb_request *request)
+{
+ struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+ struct cdns3_request *priv_req;
+ struct cdns3_trb *trb;
+ dma_addr_t trb_dma;
+ u32 togle_pcs = 1;
+ int sg_iter = 0;
+ int num_trb = 1;
+ int address;
+ u32 control;
+ int pcs;
+
+ if (num_trb > priv_ep->free_trbs) {
+ priv_ep->flags |= EP_RING_FULL;
+ return -ENOBUFS;
+ }
+
+ priv_req = to_cdns3_request(request);
+ address = priv_ep->endpoint.desc->bEndpointAddress;
+
+ priv_ep->flags |= EP_PENDING_REQUEST;
+
+ /* must allocate buffer aligned to 8 */
+ if (priv_req->flags & REQUEST_UNALIGNED)
+ trb_dma = priv_req->aligned_buf->dma;
+ else
+ trb_dma = request->dma;
+
+ trb = priv_ep->trb_pool + priv_ep->enqueue;
+ priv_req->start_trb = priv_ep->enqueue;
+ priv_req->trb = trb;
+
+ cdns3_select_ep(priv_ep->cdns3_dev, address);
+
+ /* prepare ring */
+ if ((priv_ep->enqueue + num_trb) >= (priv_ep->num_trbs - 1)) {
+ struct cdns3_trb *link_trb;
+ int doorbell, dma_index;
+ u32 ch_bit = 0;
+
+ doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
+ dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
+
+ /* Driver can't update LINK TRB if it is current processed. */
+ if (doorbell && dma_index == priv_ep->num_trbs - 1) {
+ priv_ep->flags |= EP_DEFERRED_DRDY;
+ return -ENOBUFS;
+ }
+
+ /*updating C bt in Link TRB before starting DMA*/
+ link_trb = priv_ep->trb_pool + (priv_ep->num_trbs - 1);
+ /*
+ * For TRs size equal 2 enabling TRB_CHAIN for epXin causes
+ * that DMA stuck at the LINK TRB.
+ * On the other hand, removing TRB_CHAIN for longer TRs for
+ * epXout cause that DMA stuck after handling LINK TRB.
+ * To eliminate this strange behavioral driver set TRB_CHAIN
+ * bit only for TR size > 2.
+ */
+ if (priv_ep->type == USB_ENDPOINT_XFER_ISOC ||
+ TRBS_PER_SEGMENT > 2)
+ ch_bit = TRB_CHAIN;
+
+ link_trb->control = ((priv_ep->pcs) ? TRB_CYCLE : 0) |
+ TRB_TYPE(TRB_LINK) | TRB_TOGGLE | ch_bit;
+ }
+
+ if (priv_dev->dev_ver <= DEV_VER_V2)
+ togle_pcs = cdns3_wa1_update_guard(priv_ep, trb);
+
+ /* set incorrect Cycle Bit for first trb*/
+ control = priv_ep->pcs ? 0 : TRB_CYCLE;
+
+ do {
+ u32 length;
+ u16 td_size = 0;
+
+ /* fill TRB */
+ control |= TRB_TYPE(TRB_NORMAL);
+ trb->buffer = TRB_BUFFER(trb_dma);
+
+ length = request->length;
+
+ if (likely(priv_dev->dev_ver >= DEV_VER_V2))
+ td_size = DIV_ROUND_UP(length,
+ priv_ep->endpoint.maxpacket);
+
+ trb->length = TRB_BURST_LEN(priv_ep->trb_burst_size) |
+ TRB_LEN(length);
+ if (priv_dev->gadget.speed == USB_SPEED_SUPER)
+ trb->length |= TRB_TDL_SS_SIZE(td_size);
+ else
+ control |= TRB_TDL_HS_SIZE(td_size);
+
+ pcs = priv_ep->pcs ? TRB_CYCLE : 0;
+
+ /*
+ * first trb should be prepared as last to avoid processing
+ * transfer to early
+ */
+ if (sg_iter != 0)
+ control |= pcs;
+
+ if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir) {
+ control |= TRB_IOC | TRB_ISP;
+ } else {
+ /* for last element in TD or in SG list */
+ if (sg_iter == (num_trb - 1) && sg_iter != 0)
+ control |= pcs | TRB_IOC | TRB_ISP;
+ }
+
+ if (sg_iter)
+ trb->control = control;
+ else
+ priv_req->trb->control = control;
+
+ control = 0;
+ ++sg_iter;
+ priv_req->end_trb = priv_ep->enqueue;
+ cdns3_ep_inc_enq(priv_ep);
+ trb = priv_ep->trb_pool + priv_ep->enqueue;
+ } while (sg_iter < num_trb);
+
+ trb = priv_req->trb;
+
+ priv_req->flags |= REQUEST_PENDING;
+
+ if (sg_iter == 1)
+ trb->control |= TRB_IOC | TRB_ISP;
+
+ /*
+ * Memory barrier - cycle bit must be set before other filds in trb.
+ */
+ dmb();
+
+ /* give the TD to the consumer*/
+ if (togle_pcs)
+ trb->control = trb->control ^ 1;
+
+ if (priv_dev->dev_ver <= DEV_VER_V2)
+ cdns3_wa1_tray_restore_cycle_bit(priv_dev, priv_ep);
+
+ trace_cdns3_prepare_trb(priv_ep, priv_req->trb);
+
+ /*
+ * Memory barrier - Cycle Bit must be set before trb->length and
+ * trb->buffer fields.
+ */
+ dmb();
+
+ /*
+ * For DMULT mode we can set address to transfer ring only once after
+ * enabling endpoint.
+ */
+ if (priv_ep->flags & EP_UPDATE_EP_TRBADDR) {
+ /*
+ * Until SW is not ready to handle the OUT transfer the ISO OUT
+ * Endpoint should be disabled (EP_CFG.ENABLE = 0).
+ * EP_CFG_ENABLE must be set before updating ep_traddr.
+ */
+ if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir &&
+ !(priv_ep->flags & EP_QUIRK_ISO_OUT_EN)) {
+ priv_ep->flags |= EP_QUIRK_ISO_OUT_EN;
+ cdns3_set_register_bit(&priv_dev->regs->ep_cfg,
+ EP_CFG_ENABLE);
+ }
+
+ writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma +
+ priv_req->start_trb * TRB_SIZE),
+ &priv_dev->regs->ep_traddr);
+
+ priv_ep->flags &= ~EP_UPDATE_EP_TRBADDR;
+ }
+
+ if (!priv_ep->wa1_set && !(priv_ep->flags & EP_STALLED)) {
+ trace_cdns3_ring(priv_ep);
+ /*clearing TRBERR and EP_STS_DESCMIS before seting DRDY*/
+ writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts);
+ writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
+ trace_cdns3_doorbell_epx(priv_ep->name,
+ readl(&priv_dev->regs->ep_traddr));
+ }
+
+ /* WORKAROUND for transition to L0 */
+ __cdns3_gadget_wakeup(priv_dev);
+
+ return 0;
+}
+
+void cdns3_set_hw_configuration(struct cdns3_device *priv_dev)
+{
+ struct cdns3_endpoint *priv_ep;
+ struct usb_ep *ep;
+ int val;
+
+ if (priv_dev->hw_configured_flag)
+ return;
+
+ writel(USB_CONF_CFGSET, &priv_dev->regs->usb_conf);
+ writel(EP_CMD_ERDY | EP_CMD_REQ_CMPL, &priv_dev->regs->ep_cmd);
+
+ cdns3_set_register_bit(&priv_dev->regs->usb_conf,
+ USB_CONF_U1EN | USB_CONF_U2EN);
+
+ /* wait until configuration set */
+ readl_poll_timeout_atomic(&priv_dev->regs->usb_sts, val,
+ val & USB_STS_CFGSTS_MASK, 100);
+
+ priv_dev->hw_configured_flag = 1;
+
+ list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
+ priv_ep = ep_to_cdns3_ep(ep);
+ if (priv_ep->flags & EP_ENABLED)
+ cdns3_start_all_request(priv_dev, priv_ep);
+ }
+}
+
+/**
+ * cdns3_request_handled - check whether request has been handled by DMA
+ *
+ * @priv_ep: extended endpoint object.
+ * @priv_req: request object for checking
+ *
+ * Endpoint must be selected before invoking this function.
+ *
+ * Returns false if request has not been handled by DMA, else returns true.
+ *
+ * SR - start ring
+ * ER - end ring
+ * DQ = priv_ep->dequeue - dequeue position
+ * EQ = priv_ep->enqueue - enqueue position
+ * ST = priv_req->start_trb - index of first TRB in transfer ring
+ * ET = priv_req->end_trb - index of last TRB in transfer ring
+ * CI = current_index - index of processed TRB by DMA.
+ *
+ * As first step, function checks if cycle bit for priv_req->start_trb is
+ * correct.
+ *
+ * some rules:
+ * 1. priv_ep->dequeue never exceed current_index.
+ * 2 priv_ep->enqueue never exceed priv_ep->dequeue
+ * 3. exception: priv_ep->enqueue == priv_ep->dequeue
+ * and priv_ep->free_trbs is zero.
+ * This case indicate that TR is full.
+ *
+ * Then We can split recognition into two parts:
+ * Case 1 - priv_ep->dequeue < current_index
+ * SR ... EQ ... DQ ... CI ... ER
+ * SR ... DQ ... CI ... EQ ... ER
+ *
+ * Request has been handled by DMA if ST and ET is between DQ and CI.
+ *
+ * Case 2 - priv_ep->dequeue > current_index
+ * This situation take place when CI go through the LINK TRB at the end of
+ * transfer ring.
+ * SR ... CI ... EQ ... DQ ... ER
+ *
+ * Request has been handled by DMA if ET is less then CI or
+ * ET is greater or equal DQ.
+ */
+static bool cdns3_request_handled(struct cdns3_endpoint *priv_ep,
+ struct cdns3_request *priv_req)
+{
+ struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+ struct cdns3_trb *trb = priv_req->trb;
+ int current_index = 0;
+ int handled = 0;
+ int doorbell;
+
+ current_index = cdns3_get_dma_pos(priv_dev, priv_ep);
+ doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
+
+ trb = &priv_ep->trb_pool[priv_req->start_trb];
+
+ if ((trb->control & TRB_CYCLE) != priv_ep->ccs)
+ goto finish;
+
+ if (doorbell == 1 && current_index == priv_ep->dequeue)
+ goto finish;
+
+ /* The corner case for TRBS_PER_SEGMENT equal 2). */
+ if (TRBS_PER_SEGMENT == 2 && priv_ep->type != USB_ENDPOINT_XFER_ISOC) {
+ handled = 1;
+ goto finish;
+ }
+
+ if (priv_ep->enqueue == priv_ep->dequeue &&
+ priv_ep->free_trbs == 0) {
+ handled = 1;
+ } else if (priv_ep->dequeue < current_index) {
+ if ((current_index == (priv_ep->num_trbs - 1)) &&
+ !priv_ep->dequeue)
+ goto finish;
+
+ if (priv_req->end_trb >= priv_ep->dequeue &&
+ priv_req->end_trb < current_index)
+ handled = 1;
+ } else if (priv_ep->dequeue > current_index) {
+ if (priv_req->end_trb < current_index ||
+ priv_req->end_trb >= priv_ep->dequeue)
+ handled = 1;
+ }
+
+finish:
+ trace_cdns3_request_handled(priv_req, current_index, handled);
+
+ return handled;
+}
+
+static void cdns3_transfer_completed(struct cdns3_device *priv_dev,
+ struct cdns3_endpoint *priv_ep)
+{
+ struct cdns3_request *priv_req;
+ struct usb_request *request;
+ struct cdns3_trb *trb;
+
+ while (!list_empty(&priv_ep->pending_req_list)) {
+ request = cdns3_next_request(&priv_ep->pending_req_list);
+ priv_req = to_cdns3_request(request);
+
+ /* Re-select endpoint. It could be changed by other CPU during
+ * handling usb_gadget_giveback_request.
+ */
+#ifndef __UBOOT__
+ cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
+#else
+ cdns3_select_ep(priv_dev,
+ priv_ep->endpoint.desc->bEndpointAddress);
+#endif
+
+ if (!cdns3_request_handled(priv_ep, priv_req))
+ goto prepare_next_td;
+
+ trb = priv_ep->trb_pool + priv_ep->dequeue;
+ trace_cdns3_complete_trb(priv_ep, trb);
+
+ if (trb != priv_req->trb)
+ dev_warn(priv_dev->dev,
+ "request_trb=0x%p, queue_trb=0x%p\n",
+ priv_req->trb, trb);
+
+ request->actual = TRB_LEN(le32_to_cpu(trb->length));
+ cdns3_move_deq_to_next_trb(priv_req);
+ cdns3_gadget_giveback(priv_ep, priv_req, 0);
+
+ if (priv_ep->type != USB_ENDPOINT_XFER_ISOC &&
+ TRBS_PER_SEGMENT == 2)
+ break;
+ }
+ priv_ep->flags &= ~EP_PENDING_REQUEST;
+
+prepare_next_td:
+ if (!(priv_ep->flags & EP_STALLED) &&
+ !(priv_ep->flags & EP_STALL_PENDING))
+ cdns3_start_all_request(priv_dev, priv_ep);
+}
+
+void cdns3_rearm_transfer(struct cdns3_endpoint *priv_ep, u8 rearm)
+{
+ struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+
+ cdns3_wa1_restore_cycle_bit(priv_ep);
+
+ if (rearm) {
+ trace_cdns3_ring(priv_ep);
+
+ /* Cycle Bit must be updated before arming DMA. */
+ dmb();
+ writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
+
+ __cdns3_gadget_wakeup(priv_dev);
+
+ trace_cdns3_doorbell_epx(priv_ep->name,
+ readl(&priv_dev->regs->ep_traddr));
+ }
+}
+
+/**
+ * cdns3_check_ep_interrupt_proceed - Processes interrupt related to endpoint
+ * @priv_ep: endpoint object
+ *
+ * Returns 0
+ */
+static int cdns3_check_ep_interrupt_proceed(struct cdns3_endpoint *priv_ep)
+{
+ struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+ u32 ep_sts_reg;
+
+#ifndef __UBOOT__
+ cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
+#else
+ cdns3_select_ep(priv_dev, priv_ep->endpoint.desc->bEndpointAddress);
+#endif
+
+ trace_cdns3_epx_irq(priv_dev, priv_ep);
+
+ ep_sts_reg = readl(&priv_dev->regs->ep_sts);
+ writel(ep_sts_reg, &priv_dev->regs->ep_sts);
+
+ if (ep_sts_reg & EP_STS_TRBERR) {
+ if (priv_ep->flags & EP_STALL_PENDING &&
+ !(ep_sts_reg & EP_STS_DESCMIS &&
+ priv_dev->dev_ver < DEV_VER_V2)) {
+ cdns3_ep_stall_flush(priv_ep);
+ }
+
+ /*
+ * For isochronous transfer driver completes request on
+ * IOC or on TRBERR. IOC appears only when device receive
+ * OUT data packet. If host disable stream or lost some packet
+ * then the only way to finish all queued transfer is to do it
+ * on TRBERR event.
+ */
+ if (priv_ep->type == USB_ENDPOINT_XFER_ISOC &&
+ !priv_ep->wa1_set) {
+ if (!priv_ep->dir) {
+ u32 ep_cfg = readl(&priv_dev->regs->ep_cfg);
+
+ ep_cfg &= ~EP_CFG_ENABLE;
+ writel(ep_cfg, &priv_dev->regs->ep_cfg);
+ priv_ep->flags &= ~EP_QUIRK_ISO_OUT_EN;
+ }
+ cdns3_transfer_completed(priv_dev, priv_ep);
+ } else if (!(priv_ep->flags & EP_STALLED) &&
+ !(priv_ep->flags & EP_STALL_PENDING)) {
+ if (priv_ep->flags & EP_DEFERRED_DRDY) {
+ priv_ep->flags &= ~EP_DEFERRED_DRDY;
+ cdns3_start_all_request(priv_dev, priv_ep);
+ } else {
+ cdns3_rearm_transfer(priv_ep,
+ priv_ep->wa1_set);
+ }
+ }
+ }
+
+ if ((ep_sts_reg & EP_STS_IOC) || (ep_sts_reg & EP_STS_ISP)) {
+ if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) {
+ if (ep_sts_reg & EP_STS_ISP)
+ priv_ep->flags |= EP_QUIRK_END_TRANSFER;
+ else
+ priv_ep->flags &= ~EP_QUIRK_END_TRANSFER;
+ }
+
+ cdns3_transfer_completed(priv_dev, priv_ep);
+ }
+
+ /*
+ * WA2: this condition should only be meet when
+ * priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET or
+ * priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN.
+ * In other cases this interrupt will be disabled/
+ */
+ if (ep_sts_reg & EP_STS_DESCMIS && priv_dev->dev_ver < DEV_VER_V2 &&
+ !(priv_ep->flags & EP_STALLED))
+ cdns3_wa2_descmissing_packet(priv_ep);
+
+ return 0;
+}
+
+static void cdns3_disconnect_gadget(struct cdns3_device *priv_dev)
+{
+ if (priv_dev->gadget_driver && priv_dev->gadget_driver->disconnect) {
+ spin_unlock(&priv_dev->lock);
+ priv_dev->gadget_driver->disconnect(&priv_dev->gadget);
+ spin_lock(&priv_dev->lock);
+ }
+}
+
+/**
+ * cdns3_check_usb_interrupt_proceed - Processes interrupt related to device
+ * @priv_dev: extended gadget object
+ * @usb_ists: bitmap representation of device's reported interrupts
+ * (usb_ists register value)
+ */
+static void cdns3_check_usb_interrupt_proceed(struct cdns3_device *priv_dev,
+ u32 usb_ists)
+{
+ int speed = 0;
+
+ trace_cdns3_usb_irq(priv_dev, usb_ists);
+ if (usb_ists & USB_ISTS_L1ENTI) {
+ /*
+ * WORKAROUND: CDNS3 controller has issue with hardware resuming
+ * from L1. To fix it, if any DMA transfer is pending driver
+ * must starts driving resume signal immediately.
+ */
+ if (readl(&priv_dev->regs->drbl))
+ __cdns3_gadget_wakeup(priv_dev);
+ }
+
+ /* Connection detected */
+ if (usb_ists & (USB_ISTS_CON2I | USB_ISTS_CONI)) {
+ speed = cdns3_get_speed(priv_dev);
+ priv_dev->gadget.speed = speed;
+ usb_gadget_set_state(&priv_dev->gadget, USB_STATE_POWERED);
+ cdns3_ep0_config(priv_dev);
+ }
+
+ /* Disconnection detected */
+ if (usb_ists & (USB_ISTS_DIS2I | USB_ISTS_DISI)) {
+ cdns3_disconnect_gadget(priv_dev);
+ priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
+ usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);
+ cdns3_hw_reset_eps_config(priv_dev);
+ }
+
+ if (usb_ists & (USB_ISTS_L2ENTI | USB_ISTS_U3ENTI)) {
+ if (priv_dev->gadget_driver &&
+ priv_dev->gadget_driver->suspend) {
+ spin_unlock(&priv_dev->lock);
+ priv_dev->gadget_driver->suspend(&priv_dev->gadget);
+ spin_lock(&priv_dev->lock);
+ }
+ }
+
+ if (usb_ists & (USB_ISTS_L2EXTI | USB_ISTS_U3EXTI)) {
+ if (priv_dev->gadget_driver &&
+ priv_dev->gadget_driver->resume) {
+ spin_unlock(&priv_dev->lock);
+ priv_dev->gadget_driver->resume(&priv_dev->gadget);
+ spin_lock(&priv_dev->lock);
+ }
+ }
+
+ /* reset*/
+ if (usb_ists & (USB_ISTS_UWRESI | USB_ISTS_UHRESI | USB_ISTS_U2RESI)) {
+ if (priv_dev->gadget_driver) {
+ spin_unlock(&priv_dev->lock);
+ usb_gadget_udc_reset(&priv_dev->gadget,
+ priv_dev->gadget_driver);
+ spin_lock(&priv_dev->lock);
+
+ /*read again to check the actual speed*/
+ speed = cdns3_get_speed(priv_dev);
+ priv_dev->gadget.speed = speed;
+ cdns3_hw_reset_eps_config(priv_dev);
+ cdns3_ep0_config(priv_dev);
+ }
+ }
+}
+
+/**
+ * cdns3_device_irq_handler- interrupt handler for device part of controller
+ *
+ * @irq: irq number for cdns3 core device
+ * @data: structure of cdns3
+ *
+ * Returns IRQ_HANDLED or IRQ_NONE
+ */
+static irqreturn_t cdns3_device_irq_handler(int irq, void *data)
+{
+ struct cdns3_device *priv_dev;
+ struct cdns3 *cdns = data;
+ irqreturn_t ret = IRQ_NONE;
+ u32 reg;
+
+ priv_dev = cdns->gadget_dev;
+
+ /* check USB device interrupt */
+ reg = readl(&priv_dev->regs->usb_ists);
+ if (reg) {
+ /* After masking interrupts the new interrupts won't be
+ * reported in usb_ists/ep_ists. In order to not lose some
+ * of them driver disables only detected interrupts.
+ * They will be enabled ASAP after clearing source of
+ * interrupt. This an unusual behavior only applies to
+ * usb_ists register.
+ */
+ reg = ~reg & readl(&priv_dev->regs->usb_ien);
+ /* mask deferred interrupt. */
+ writel(reg, &priv_dev->regs->usb_ien);
+ ret = IRQ_WAKE_THREAD;
+ }
+
+ /* check endpoint interrupt */
+ reg = readl(&priv_dev->regs->ep_ists);
+ if (reg) {
+ writel(0, &priv_dev->regs->ep_ien);
+ ret = IRQ_WAKE_THREAD;
+ }
+
+ return ret;
+}
+
+/**
+ * cdns3_device_thread_irq_handler- interrupt handler for device part
+ * of controller
+ *
+ * @irq: irq number for cdns3 core device
+ * @data: structure of cdns3
+ *
+ * Returns IRQ_HANDLED or IRQ_NONE
+ */
+static irqreturn_t cdns3_device_thread_irq_handler(int irq, void *data)
+{
+ struct cdns3_device *priv_dev;
+ struct cdns3 *cdns = data;
+ irqreturn_t ret = IRQ_NONE;
+ unsigned long flags;
+ int bit;
+ u32 reg;
+
+ priv_dev = cdns->gadget_dev;
+ spin_lock_irqsave(&priv_dev->lock, flags);
+
+ reg = readl(&priv_dev->regs->usb_ists);
+ if (reg) {
+ writel(reg, &priv_dev->regs->usb_ists);
+ writel(USB_IEN_INIT, &priv_dev->regs->usb_ien);
+ cdns3_check_usb_interrupt_proceed(priv_dev, reg);
+ ret = IRQ_HANDLED;
+ }
+
+ reg = readl(&priv_dev->regs->ep_ists);
+
+ /* handle default endpoint OUT */
+ if (reg & EP_ISTS_EP_OUT0) {
+ cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_OUT);
+ ret = IRQ_HANDLED;
+ }
+
+ /* handle default endpoint IN */
+ if (reg & EP_ISTS_EP_IN0) {
+ cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_IN);
+ ret = IRQ_HANDLED;
+ }
+
+ /* check if interrupt from non default endpoint, if no exit */
+ reg &= ~(EP_ISTS_EP_OUT0 | EP_ISTS_EP_IN0);
+ if (!reg)
+ goto irqend;
+
+ for_each_set_bit(bit, (unsigned long *)®,
+ sizeof(u32) * BITS_PER_BYTE) {
+ cdns3_check_ep_interrupt_proceed(priv_dev->eps[bit]);
+ ret = IRQ_HANDLED;
+ }
+
+irqend:
+ writel(~0, &priv_dev->regs->ep_ien);
+ spin_unlock_irqrestore(&priv_dev->lock, flags);
+
+ return ret;
+}
+
+/**
+ * cdns3_ep_onchip_buffer_reserve - Try to reserve onchip buf for EP
+ *
+ * The real reservation will occur during write to EP_CFG register,
+ * this function is used to check if the 'size' reservation is allowed.
+ *
+ * @priv_dev: extended gadget object
+ * @size: the size (KB) for EP would like to allocate
+ * @is_in: endpoint direction
+ *
+ * Return 0 if the required size can met or negative value on failure
+ */
+static int cdns3_ep_onchip_buffer_reserve(struct cdns3_device *priv_dev,
+ int size, int is_in)
+{
+ int remained;
+
+ /* 2KB are reserved for EP0*/
+ remained = priv_dev->onchip_buffers - priv_dev->onchip_used_size - 2;
+
+ if (is_in) {
+ if (remained < size)
+ return -EPERM;
+
+ priv_dev->onchip_used_size += size;
+ } else {
+ int required;
+
+ /**
+ * ALL OUT EPs are shared the same chunk onchip memory, so
+ * driver checks if it already has assigned enough buffers
+ */
+ if (priv_dev->out_mem_is_allocated >= size)
+ return 0;
+
+ required = size - priv_dev->out_mem_is_allocated;
+
+ if (required > remained)
+ return -EPERM;
+
+ priv_dev->out_mem_is_allocated += required;
+ priv_dev->onchip_used_size += required;
+ }
+
+ return 0;
+}
+
+void cdns3_configure_dmult(struct cdns3_device *priv_dev,
+ struct cdns3_endpoint *priv_ep)
+{
+ struct cdns3_usb_regs __iomem *regs = priv_dev->regs;
+
+ /* For dev_ver > DEV_VER_V2 DMULT is configured per endpoint */
+ if (priv_dev->dev_ver <= DEV_VER_V2)
+ writel(USB_CONF_DMULT, ®s->usb_conf);
+
+ if (priv_dev->dev_ver == DEV_VER_V2)
+ writel(USB_CONF2_EN_TDL_TRB, ®s->usb_conf2);
+
+ if (priv_dev->dev_ver >= DEV_VER_V3 && priv_ep) {
+ u32 mask;
+
+ if (priv_ep->dir)
+ mask = BIT(priv_ep->num + 16);
+ else
+ mask = BIT(priv_ep->num);
+
+ if (priv_ep->type != USB_ENDPOINT_XFER_ISOC) {
+ cdns3_set_register_bit(®s->tdl_from_trb, mask);
+ cdns3_set_register_bit(®s->tdl_beh, mask);
+ cdns3_set_register_bit(®s->tdl_beh2, mask);
+ cdns3_set_register_bit(®s->dma_adv_td, mask);
+ }
+
+ if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir)
+ cdns3_set_register_bit(®s->tdl_from_trb, mask);
+
+ cdns3_set_register_bit(®s->dtrans, mask);
+ }
+}
+
+/**
+ * cdns3_ep_config Configure hardware endpoint
+ * @priv_ep: extended endpoint object
+ */
+void cdns3_ep_config(struct cdns3_endpoint *priv_ep)
+{
+ bool is_iso_ep = (priv_ep->type == USB_ENDPOINT_XFER_ISOC);
+ struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+ u32 bEndpointAddress = priv_ep->num | priv_ep->dir;
+ u32 max_packet_size = 0;
+ u8 maxburst = 0;
+ u32 ep_cfg = 0;
+ u8 buffering;
+ u8 mult = 0;
+ int ret;
+
+ buffering = CDNS3_EP_BUF_SIZE - 1;
+
+ cdns3_configure_dmult(priv_dev, priv_ep);
+
+ switch (priv_ep->type) {
+ case USB_ENDPOINT_XFER_INT:
+ ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_INT);
+
+ if ((priv_dev->dev_ver == DEV_VER_V2 && !priv_ep->dir) ||
+ priv_dev->dev_ver > DEV_VER_V2)
+ ep_cfg |= EP_CFG_TDL_CHK;
+ break;
+ case USB_ENDPOINT_XFER_BULK:
+ ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_BULK);
+
+ if ((priv_dev->dev_ver == DEV_VER_V2 && !priv_ep->dir) ||
+ priv_dev->dev_ver > DEV_VER_V2)
+ ep_cfg |= EP_CFG_TDL_CHK;
+ break;
+ default:
+ ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_ISOC);
+ mult = CDNS3_EP_ISO_HS_MULT - 1;
+ buffering = mult + 1;
+ }
+
+ switch (priv_dev->gadget.speed) {
+ case USB_SPEED_FULL:
+ max_packet_size = is_iso_ep ? 1023 : 64;
+ break;
+ case USB_SPEED_HIGH:
+ max_packet_size = is_iso_ep ? 1024 : 512;
+ break;
+ case USB_SPEED_SUPER:
+ /* It's limitation that driver assumes in driver. */
+ mult = 0;
+ max_packet_size = 1024;
+ if (priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
+ maxburst = CDNS3_EP_ISO_SS_BURST - 1;
+ buffering = (mult + 1) *
+ (maxburst + 1);
+
+ if (priv_ep->interval > 1)
+ buffering++;
+ } else {
+ maxburst = CDNS3_EP_BUF_SIZE - 1;
+ }
+ break;
+ default:
+ /* all other speed are not supported */
+ return;
+ }
+
+ if (max_packet_size == 1024)
+ priv_ep->trb_burst_size = 128;
+ else if (max_packet_size >= 512)
+ priv_ep->trb_burst_size = 64;
+ else
+ priv_ep->trb_burst_size = 16;
+
+ ret = cdns3_ep_onchip_buffer_reserve(priv_dev, buffering + 1,
+ !!priv_ep->dir);
+ if (ret) {
+ dev_err(priv_dev->dev, "onchip mem is full, ep is invalid\n");
+ return;
+ }
+
+ ep_cfg |= EP_CFG_MAXPKTSIZE(max_packet_size) |
+ EP_CFG_MULT(mult) |
+ EP_CFG_BUFFERING(buffering) |
+ EP_CFG_MAXBURST(maxburst);
+
+ cdns3_select_ep(priv_dev, bEndpointAddress);
+ writel(ep_cfg, &priv_dev->regs->ep_cfg);
+
+ dev_dbg(priv_dev->dev, "Configure %s: with val %08x\n",
+ priv_ep->name, ep_cfg);
+}
+
+/* Find correct direction for HW endpoint according to description */
+static int cdns3_ep_dir_is_correct(struct usb_endpoint_descriptor *desc,
+ struct cdns3_endpoint *priv_ep)
+{
+ return (priv_ep->endpoint.caps.dir_in && usb_endpoint_dir_in(desc)) ||
+ (priv_ep->endpoint.caps.dir_out && usb_endpoint_dir_out(desc));
+}
+
+static struct
+cdns3_endpoint *cdns3_find_available_ep(struct cdns3_device *priv_dev,
+ struct usb_endpoint_descriptor *desc)
+{
+ struct usb_ep *ep;
+ struct cdns3_endpoint *priv_ep;
+
+ list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
+ unsigned long num;
+ /* ep name pattern likes epXin or epXout */
+ char c[2] = {ep->name[2], '\0'};
+
+ num = simple_strtoul(c, NULL, 10);
+
+ priv_ep = ep_to_cdns3_ep(ep);
+ if (cdns3_ep_dir_is_correct(desc, priv_ep)) {
+ if (!(priv_ep->flags & EP_CLAIMED)) {
+ priv_ep->num = num;
+ return priv_ep;
+ }
+ }
+ }
+
+ return ERR_PTR(-ENOENT);
+}
+
+/*
+ * Cadence IP has one limitation that all endpoints must be configured
+ * (Type & MaxPacketSize) before setting configuration through hardware
+ * register, it means we can't change endpoints configuration after
+ * set_configuration.
+ *
+ * This function set EP_CLAIMED flag which is added when the gadget driver
+ * uses usb_ep_autoconfig to configure specific endpoint;
+ * When the udc driver receives set_configurion request,
+ * it goes through all claimed endpoints, and configure all endpoints
+ * accordingly.
+ *
+ * At usb_ep_ops.enable/disable, we only enable and disable endpoint through
+ * ep_cfg register which can be changed after set_configuration, and do
+ * some software operation accordingly.
+ */
+static struct
+usb_ep *cdns3_gadget_match_ep(struct usb_gadget *gadget,
+ struct usb_endpoint_descriptor *desc,
+ struct usb_ss_ep_comp_descriptor *comp_desc)
+{
+ struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
+ struct cdns3_endpoint *priv_ep;
+ unsigned long flags;
+
+ priv_ep = cdns3_find_available_ep(priv_dev, desc);
+ if (IS_ERR(priv_ep)) {
+ dev_err(priv_dev->dev, "no available ep\n");
+ return NULL;
+ }
+
+ dev_dbg(priv_dev->dev, "match endpoint: %s\n", priv_ep->name);
+
+ spin_lock_irqsave(&priv_dev->lock, flags);
+ priv_ep->endpoint.desc = desc;
+ priv_ep->dir = usb_endpoint_dir_in(desc) ? USB_DIR_IN : USB_DIR_OUT;
+ priv_ep->type = usb_endpoint_type(desc);
+ priv_ep->flags |= EP_CLAIMED;
+ priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
+
+ spin_unlock_irqrestore(&priv_dev->lock, flags);
+ return &priv_ep->endpoint;
+}
+
+/**
+ * cdns3_gadget_ep_alloc_request Allocates request
+ * @ep: endpoint object associated with request
+ * @gfp_flags: gfp flags
+ *
+ * Returns allocated request address, NULL on allocation error
+ */
+struct usb_request *cdns3_gadget_ep_alloc_request(struct usb_ep *ep,
+ gfp_t gfp_flags)
+{
+ struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
+ struct cdns3_request *priv_req;
+
+ priv_req = kzalloc(sizeof(*priv_req), gfp_flags);
+ if (!priv_req)
+ return NULL;
+
+ priv_req->priv_ep = priv_ep;
+
+ trace_cdns3_alloc_request(priv_req);
+ return &priv_req->request;
+}
+
+/**
+ * cdns3_gadget_ep_free_request Free memory occupied by request
+ * @ep: endpoint object associated with request
+ * @request: request to free memory
+ */
+void cdns3_gadget_ep_free_request(struct usb_ep *ep,
+ struct usb_request *request)
+{
+ struct cdns3_request *priv_req = to_cdns3_request(request);
+
+ if (priv_req->aligned_buf)
+ priv_req->aligned_buf->in_use = 0;
+
+ trace_cdns3_free_request(priv_req);
+ kfree(priv_req);
+}
+
+/**
+ * cdns3_gadget_ep_enable Enable endpoint
+ * @ep: endpoint object
+ * @desc: endpoint descriptor
+ *
+ * Returns 0 on success, error code elsewhere
+ */
+static int cdns3_gadget_ep_enable(struct usb_ep *ep,
+ const struct usb_endpoint_descriptor *desc)
+{
+ struct cdns3_endpoint *priv_ep;
+ struct cdns3_device *priv_dev;
+ u32 reg = EP_STS_EN_TRBERREN;
+ u32 bEndpointAddress;
+ unsigned long flags;
+ int enable = 1;
+ int ret;
+ int val;
+
+ priv_ep = ep_to_cdns3_ep(ep);
+ priv_dev = priv_ep->cdns3_dev;
+
+ if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
+ dev_dbg(priv_dev->dev, "usbss: invalid parameters\n");
+ return -EINVAL;
+ }
+
+ if (!desc->wMaxPacketSize) {
+ dev_err(priv_dev->dev, "usbss: missing wMaxPacketSize\n");
+ return -EINVAL;
+ }
+
+ if (WARN_ON(priv_ep->flags & EP_ENABLED))
+ return 0;
+
+ spin_lock_irqsave(&priv_dev->lock, flags);
+
+ priv_ep->endpoint.desc = desc;
+ priv_ep->type = usb_endpoint_type(desc);
+ priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
+
+ if (priv_ep->interval > ISO_MAX_INTERVAL &&
+ priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
+ dev_err(priv_dev->dev, "Driver is limited to %d period\n",
+ ISO_MAX_INTERVAL);
+
+ ret = -EINVAL;
+ goto exit;
+ }
+
+ ret = cdns3_allocate_trb_pool(priv_ep);
+
+ if (ret)
+ goto exit;
+
+ bEndpointAddress = priv_ep->num | priv_ep->dir;
+ cdns3_select_ep(priv_dev, bEndpointAddress);
+
+ trace_cdns3_gadget_ep_enable(priv_ep);
+
+ writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
+
+ ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
+ !(val & (EP_CMD_CSTALL | EP_CMD_EPRST)),
+ 1000);
+
+ if (unlikely(ret)) {
+ cdns3_free_trb_pool(priv_ep);
+ ret = -EINVAL;
+ goto exit;
+ }
+
+ /* enable interrupt for selected endpoint */
+ cdns3_set_register_bit(&priv_dev->regs->ep_ien,
+ BIT(cdns3_ep_addr_to_index(bEndpointAddress)));
+
+ if (priv_dev->dev_ver < DEV_VER_V2)
+ cdns3_wa2_enable_detection(priv_dev, priv_ep, reg);
+
+ writel(reg, &priv_dev->regs->ep_sts_en);
+
+ /*
+ * For some versions of controller at some point during ISO OUT traffic
+ * DMA reads Transfer Ring for the EP which has never got doorbell.
+ * This issue was detected only on simulation, but to avoid this issue
+ * driver add protection against it. To fix it driver enable ISO OUT
+ * endpoint before setting DRBL. This special treatment of ISO OUT
+ * endpoints are recommended by controller specification.
+ */
+ if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir)
+ enable = 0;
+
+ if (enable)
+ cdns3_set_register_bit(&priv_dev->regs->ep_cfg, EP_CFG_ENABLE);
+
+ ep->desc = desc;
+ priv_ep->flags &= ~(EP_PENDING_REQUEST | EP_STALLED | EP_STALL_PENDING |
+ EP_QUIRK_ISO_OUT_EN | EP_QUIRK_EXTRA_BUF_EN);
+ priv_ep->flags |= EP_ENABLED | EP_UPDATE_EP_TRBADDR;
+ priv_ep->wa1_set = 0;
+ priv_ep->enqueue = 0;
+ priv_ep->dequeue = 0;
+ reg = readl(&priv_dev->regs->ep_sts);
+ priv_ep->pcs = !!EP_STS_CCS(reg);
+ priv_ep->ccs = !!EP_STS_CCS(reg);
+ /* one TRB is reserved for link TRB used in DMULT mode*/
+ priv_ep->free_trbs = priv_ep->num_trbs - 1;
+exit:
+ spin_unlock_irqrestore(&priv_dev->lock, flags);
+
+ return ret;
+}
+
+/**
+ * cdns3_gadget_ep_disable Disable endpoint
+ * @ep: endpoint object
+ *
+ * Returns 0 on success, error code elsewhere
+ */
+static int cdns3_gadget_ep_disable(struct usb_ep *ep)
+{
+ struct cdns3_endpoint *priv_ep;
+ struct cdns3_request *priv_req;
+ struct cdns3_device *priv_dev;
+ struct usb_request *request;
+ unsigned long flags;
+ int ret = 0;
+ u32 ep_cfg;
+ int val;
+
+ if (!ep) {
+ pr_err("usbss: invalid parameters\n");
+ return -EINVAL;
+ }
+
+ priv_ep = ep_to_cdns3_ep(ep);
+ priv_dev = priv_ep->cdns3_dev;
+
+ if (WARN_ON(!(priv_ep->flags & EP_ENABLED)))
+ return 0;
+
+ spin_lock_irqsave(&priv_dev->lock, flags);
+
+ trace_cdns3_gadget_ep_disable(priv_ep);
+
+ cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
+
+ ep_cfg = readl(&priv_dev->regs->ep_cfg);
+ ep_cfg &= ~EP_CFG_ENABLE;
+ writel(ep_cfg, &priv_dev->regs->ep_cfg);
+
+ /**
+ * Driver needs some time before resetting endpoint.
+ * It need waits for clearing DBUSY bit or for timeout expired.
+ * 10us is enough time for controller to stop transfer.
+ */
+ readl_poll_timeout_atomic(&priv_dev->regs->ep_sts, val,
+ !(val & EP_STS_DBUSY), 10);
+ writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
+
+ readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
+ !(val & (EP_CMD_CSTALL | EP_CMD_EPRST)),
+ 1000);
+ if (unlikely(ret))
+ dev_err(priv_dev->dev, "Timeout: %s resetting failed.\n",
+ priv_ep->name);
+
+ while (!list_empty(&priv_ep->pending_req_list)) {
+ request = cdns3_next_request(&priv_ep->pending_req_list);
+
+ cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
+ -ESHUTDOWN);
+ }
+
+ while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
+ priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
+
+ kfree(priv_req->request.buf);
+ cdns3_gadget_ep_free_request(&priv_ep->endpoint,
+ &priv_req->request);
+ list_del_init(&priv_req->list);
+ --priv_ep->wa2_counter;
+ }
+
+ while (!list_empty(&priv_ep->deferred_req_list)) {
+ request = cdns3_next_request(&priv_ep->deferred_req_list);
+
+ cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
+ -ESHUTDOWN);
+ }
+
+ priv_ep->descmis_req = NULL;
+
+ ep->desc = NULL;
+ priv_ep->flags &= ~EP_ENABLED;
+
+ spin_unlock_irqrestore(&priv_dev->lock, flags);
+
+ return ret;
+}
+
+/**
+ * cdns3_gadget_ep_queue Transfer data on endpoint
+ * @ep: endpoint object
+ * @request: request object
+ * @gfp_flags: gfp flags
+ *
+ * Returns 0 on success, error code elsewhere
+ */
+static int __cdns3_gadget_ep_queue(struct usb_ep *ep,
+ struct usb_request *request,
+ gfp_t gfp_flags)
+{
+ struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
+ struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+ struct cdns3_request *priv_req;
+ int ret = 0;
+
+ request->actual = 0;
+ request->status = -EINPROGRESS;
+ priv_req = to_cdns3_request(request);
+ trace_cdns3_ep_queue(priv_req);
+
+ if (priv_dev->dev_ver < DEV_VER_V2) {
+ ret = cdns3_wa2_gadget_ep_queue(priv_dev, priv_ep,
+ priv_req);
+
+ if (ret == EINPROGRESS)
+ return 0;
+ }
+
+ ret = cdns3_prepare_aligned_request_buf(priv_req);
+ if (ret < 0)
+ return ret;
+
+ ret = usb_gadget_map_request(&priv_dev->gadget, request,
+ usb_endpoint_dir_in(ep->desc));
+ if (ret)
+ return ret;
+
+ list_add_tail(&request->list, &priv_ep->deferred_req_list);
+
+ /*
+ * If hardware endpoint configuration has not been set yet then
+ * just queue request in deferred list. Transfer will be started in
+ * cdns3_set_hw_configuration.
+ */
+ if (priv_dev->hw_configured_flag && !(priv_ep->flags & EP_STALLED) &&
+ !(priv_ep->flags & EP_STALL_PENDING))
+ cdns3_start_all_request(priv_dev, priv_ep);
+
+ return 0;
+}
+
+static int cdns3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
+ gfp_t gfp_flags)
+{
+ struct usb_request *zlp_request;
+ struct cdns3_endpoint *priv_ep;
+ struct cdns3_device *priv_dev;
+ unsigned long flags;
+ int ret;
+
+ if (!request || !ep)
+ return -EINVAL;
+
+ priv_ep = ep_to_cdns3_ep(ep);
+ priv_dev = priv_ep->cdns3_dev;
+
+ spin_lock_irqsave(&priv_dev->lock, flags);
+
+ ret = __cdns3_gadget_ep_queue(ep, request, gfp_flags);
+
+ if (ret == 0 && request->zero && request->length &&
+ (request->length % ep->maxpacket == 0)) {
+ struct cdns3_request *priv_req;
+
+ zlp_request = cdns3_gadget_ep_alloc_request(ep, GFP_ATOMIC);
+ zlp_request->buf = priv_dev->zlp_buf;
+ zlp_request->length = 0;
+
+ priv_req = to_cdns3_request(zlp_request);
+ priv_req->flags |= REQUEST_ZLP;
+
+ dev_dbg(priv_dev->dev, "Queuing ZLP for endpoint: %s\n",
+ priv_ep->name);
+ ret = __cdns3_gadget_ep_queue(ep, zlp_request, gfp_flags);
+ }
+
+ spin_unlock_irqrestore(&priv_dev->lock, flags);
+ return ret;
+}
+
+/**
+ * cdns3_gadget_ep_dequeue Remove request from transfer queue
+ * @ep: endpoint object associated with request
+ * @request: request object
+ *
+ * Returns 0 on success, error code elsewhere
+ */
+int cdns3_gadget_ep_dequeue(struct usb_ep *ep,
+ struct usb_request *request)
+{
+ struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
+ struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+ struct usb_request *req, *req_temp;
+ struct cdns3_request *priv_req;
+ struct cdns3_trb *link_trb;
+ unsigned long flags;
+ int ret = 0;
+
+ if (!ep || !request || !ep->desc)
+ return -EINVAL;
+
+ spin_lock_irqsave(&priv_dev->lock, flags);
+
+ priv_req = to_cdns3_request(request);
+
+ trace_cdns3_ep_dequeue(priv_req);
+
+ cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
+
+ list_for_each_entry_safe(req, req_temp, &priv_ep->pending_req_list,
+ list) {
+ if (request == req)
+ goto found;
+ }
+
+ list_for_each_entry_safe(req, req_temp, &priv_ep->deferred_req_list,
+ list) {
+ if (request == req)
+ goto found;
+ }
+
+ goto not_found;
+
+found:
+
+ if (priv_ep->wa1_trb == priv_req->trb)
+ cdns3_wa1_restore_cycle_bit(priv_ep);
+
+ link_trb = priv_req->trb;
+ cdns3_move_deq_to_next_trb(priv_req);
+ cdns3_gadget_giveback(priv_ep, priv_req, -ECONNRESET);
+
+ /* Update ring */
+ request = cdns3_next_request(&priv_ep->deferred_req_list);
+ if (request) {
+ priv_req = to_cdns3_request(request);
+
+ link_trb->buffer = TRB_BUFFER(priv_ep->trb_pool_dma +
+ (priv_req->start_trb * TRB_SIZE));
+ link_trb->control = (link_trb->control & TRB_CYCLE) |
+ TRB_TYPE(TRB_LINK) | TRB_CHAIN | TRB_TOGGLE;
+ } else {
+ priv_ep->flags |= EP_UPDATE_EP_TRBADDR;
+ }
+
+not_found:
+ spin_unlock_irqrestore(&priv_dev->lock, flags);
+ return ret;
+}
+
+/**
+ * __cdns3_gadget_ep_set_halt Sets stall on selected endpoint
+ * Should be called after acquiring spin_lock and selecting ep
+ * @ep: endpoint object to set stall on.
+ */
+void __cdns3_gadget_ep_set_halt(struct cdns3_endpoint *priv_ep)
+{
+ struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+
+ trace_cdns3_halt(priv_ep, 1, 0);
+
+ if (!(priv_ep->flags & EP_STALLED)) {
+ u32 ep_sts_reg = readl(&priv_dev->regs->ep_sts);
+
+ if (!(ep_sts_reg & EP_STS_DBUSY))
+ cdns3_ep_stall_flush(priv_ep);
+ else
+ priv_ep->flags |= EP_STALL_PENDING;
+ }
+}
+
+/**
+ * __cdns3_gadget_ep_clear_halt Clears stall on selected endpoint
+ * Should be called after acquiring spin_lock and selecting ep
+ * @ep: endpoint object to clear stall on
+ */
+int __cdns3_gadget_ep_clear_halt(struct cdns3_endpoint *priv_ep)
+{
+ struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+ struct usb_request *request;
+ int ret = 0;
+ int val;
+
+ trace_cdns3_halt(priv_ep, 0, 0);
+
+ writel(EP_CMD_CSTALL | EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
+
+ /* wait for EPRST cleared */
+ readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
+ !(val & EP_CMD_EPRST), 100);
+ if (ret)
+ return -EINVAL;
+
+ priv_ep->flags &= ~(EP_STALLED | EP_STALL_PENDING);
+
+ request = cdns3_next_request(&priv_ep->pending_req_list);
+
+ if (request)
+ cdns3_rearm_transfer(priv_ep, 1);
+
+ cdns3_start_all_request(priv_dev, priv_ep);
+ return ret;
+}
+
+/**
+ * cdns3_gadget_ep_set_halt Sets/clears stall on selected endpoint
+ * @ep: endpoint object to set/clear stall on
+ * @value: 1 for set stall, 0 for clear stall
+ *
+ * Returns 0 on success, error code elsewhere
+ */
+int cdns3_gadget_ep_set_halt(struct usb_ep *ep, int value)
+{
+ struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
+ struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
+ unsigned long flags;
+ int ret = 0;
+
+ if (!(priv_ep->flags & EP_ENABLED))
+ return -EPERM;
+
+ spin_lock_irqsave(&priv_dev->lock, flags);
+
+ cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
+
+ if (!value) {
+ priv_ep->flags &= ~EP_WEDGE;
+ ret = __cdns3_gadget_ep_clear_halt(priv_ep);
+ } else {
+ __cdns3_gadget_ep_set_halt(priv_ep);
+ }
+
+ spin_unlock_irqrestore(&priv_dev->lock, flags);
+
+ return ret;
+}
+
+extern const struct usb_ep_ops cdns3_gadget_ep0_ops;
+
+static const struct usb_ep_ops cdns3_gadget_ep_ops = {
+ .enable = cdns3_gadget_ep_enable,
+ .disable = cdns3_gadget_ep_disable,
+ .alloc_request = cdns3_gadget_ep_alloc_request,
+ .free_request = cdns3_gadget_ep_free_request,
+ .queue = cdns3_gadget_ep_queue,
+ .dequeue = cdns3_gadget_ep_dequeue,
+ .set_halt = cdns3_gadget_ep_set_halt,
+ .set_wedge = cdns3_gadget_ep_set_wedge,
+};
+
+/**
+ * cdns3_gadget_get_frame Returns number of actual ITP frame
+ * @gadget: gadget object
+ *
+ * Returns number of actual ITP frame
+ */
+static int cdns3_gadget_get_frame(struct usb_gadget *gadget)
+{
+ struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
+
+ return readl(&priv_dev->regs->usb_itpn);
+}
+
+int __cdns3_gadget_wakeup(struct cdns3_device *priv_dev)
+{
+ enum usb_device_speed speed;
+
+ speed = cdns3_get_speed(priv_dev);
+
+ if (speed >= USB_SPEED_SUPER)
+ return 0;
+
+ /* Start driving resume signaling to indicate remote wakeup. */
+ writel(USB_CONF_LGO_L0, &priv_dev->regs->usb_conf);
+
+ return 0;
+}
+
+static int cdns3_gadget_wakeup(struct usb_gadget *gadget)
+{
+ struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
+ unsigned long flags;
+ int ret = 0;
+
+ spin_lock_irqsave(&priv_dev->lock, flags);
+ ret = __cdns3_gadget_wakeup(priv_dev);
+ spin_unlock_irqrestore(&priv_dev->lock, flags);
+ return ret;
+}
+
+static int cdns3_gadget_set_selfpowered(struct usb_gadget *gadget,
+ int is_selfpowered)
+{
+ struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
+ unsigned long flags;
+
+ spin_lock_irqsave(&priv_dev->lock, flags);
+ priv_dev->is_selfpowered = !!is_selfpowered;
+ spin_unlock_irqrestore(&priv_dev->lock, flags);
+ return 0;
+}
+
+static int cdns3_gadget_pullup(struct usb_gadget *gadget, int is_on)
+{
+ struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
+
+ if (is_on)
+ writel(USB_CONF_DEVEN, &priv_dev->regs->usb_conf);
+ else
+ writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);
+
+ return 0;
+}
+
+static void cdns3_gadget_config(struct cdns3_device *priv_dev)
+{
+ struct cdns3_usb_regs __iomem *regs = priv_dev->regs;
+ u32 reg;
+
+ cdns3_ep0_config(priv_dev);
+
+ /* enable interrupts for endpoint 0 (in and out) */
+ writel(EP_IEN_EP_OUT0 | EP_IEN_EP_IN0, ®s->ep_ien);
+
+ /*
+ * Driver needs to modify LFPS minimal U1 Exit time for DEV_VER_TI_V1
+ * revision of controller.
+ */
+ if (priv_dev->dev_ver == DEV_VER_TI_V1) {
+ reg = readl(®s->dbg_link1);
+
+ reg &= ~DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_MASK;
+ reg |= DBG_LINK1_LFPS_MIN_GEN_U1_EXIT(0x55) |
+ DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_SET;
+ writel(reg, ®s->dbg_link1);
+ }
+
+ /*
+ * By default some platforms has set protected access to memory.
+ * This cause problem with cache, so driver restore non-secure
+ * access to memory.
+ */
+ reg = readl(®s->dma_axi_ctrl);
+ reg |= DMA_AXI_CTRL_MARPROT(DMA_AXI_CTRL_NON_SECURE) |
+ DMA_AXI_CTRL_MAWPROT(DMA_AXI_CTRL_NON_SECURE);
+ writel(reg, ®s->dma_axi_ctrl);
+
+ /* enable generic interrupt*/
+ writel(USB_IEN_INIT, ®s->usb_ien);
+ writel(USB_CONF_CLK2OFFDS | USB_CONF_L1DS, ®s->usb_conf);
+
+ cdns3_configure_dmult(priv_dev, NULL);
+
+ cdns3_gadget_pullup(&priv_dev->gadget, 1);
+}
+
+/**
+ * cdns3_gadget_udc_start Gadget start
+ * @gadget: gadget object
+ * @driver: driver which operates on this gadget
+ *
+ * Returns 0 on success, error code elsewhere
+ */
+static int cdns3_gadget_udc_start(struct usb_gadget *gadget,
+ struct usb_gadget_driver *driver)
+{
+ struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
+ unsigned long flags;
+
+ spin_lock_irqsave(&priv_dev->lock, flags);
+ priv_dev->gadget_driver = driver;
+ cdns3_gadget_config(priv_dev);
+ spin_unlock_irqrestore(&priv_dev->lock, flags);
+ return 0;
+}
+
+/**
+ * cdns3_gadget_udc_stop Stops gadget
+ * @gadget: gadget object
+ *
+ * Returns 0
+ */
+static int cdns3_gadget_udc_stop(struct usb_gadget *gadget)
+{
+ struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
+ struct cdns3_endpoint *priv_ep;
+ u32 bEndpointAddress;
+ struct usb_ep *ep;
+ int ret = 0;
+ int val;
+
+ priv_dev->gadget_driver = NULL;
+
+ priv_dev->onchip_used_size = 0;
+ priv_dev->out_mem_is_allocated = 0;
+ priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
+
+ list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
+ priv_ep = ep_to_cdns3_ep(ep);
+ bEndpointAddress = priv_ep->num | priv_ep->dir;
+ cdns3_select_ep(priv_dev, bEndpointAddress);
+ writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
+ readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
+ !(val & EP_CMD_EPRST), 100);
+ }
+
+ /* disable interrupt for device */
+ writel(0, &priv_dev->regs->usb_ien);
+ writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);
+
+ return ret;
+}
+
+static void cdns3_gadget_udc_set_speed(struct usb_gadget *gadget,
+ enum usb_device_speed speed)
+{
+ struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
+
+ switch (speed) {
+ case USB_SPEED_FULL:
+ writel(USB_CONF_SFORCE_FS, &priv_dev->regs->usb_conf);
+ writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf);
+ break;
+ case USB_SPEED_HIGH:
+ writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf);
+ break;
+ case USB_SPEED_SUPER:
+ break;
+ default:
+ dev_err(cdns->dev, "invalid speed parameter %d\n",
+ speed);
+ }
+
+ priv_dev->gadget.speed = speed;
+}
+
+static const struct usb_gadget_ops cdns3_gadget_ops = {
+ .get_frame = cdns3_gadget_get_frame,
+ .wakeup = cdns3_gadget_wakeup,
+ .set_selfpowered = cdns3_gadget_set_selfpowered,
+ .pullup = cdns3_gadget_pullup,
+ .udc_start = cdns3_gadget_udc_start,
+ .udc_stop = cdns3_gadget_udc_stop,
+ .match_ep = cdns3_gadget_match_ep,
+ .udc_set_speed = cdns3_gadget_udc_set_speed,
+};
+
+static void cdns3_free_all_eps(struct cdns3_device *priv_dev)
+{
+ int i;
+
+ /* ep0 OUT point to ep0 IN. */
+ priv_dev->eps[16] = NULL;
+
+ for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++)
+ if (priv_dev->eps[i]) {
+ cdns3_free_trb_pool(priv_dev->eps[i]);
+ devm_kfree(priv_dev->dev, priv_dev->eps[i]);
+ }
+}
+
+/**
+ * cdns3_init_eps Initializes software endpoints of gadget
+ * @cdns3: extended gadget object
+ *
+ * Returns 0 on success, error code elsewhere
+ */
+static int cdns3_init_eps(struct cdns3_device *priv_dev)
+{
+ u32 ep_enabled_reg, iso_ep_reg;
+ struct cdns3_endpoint *priv_ep;
+ int ep_dir, ep_number;
+ u32 ep_mask;
+ int ret = 0;
+ int i;
+
+ /* Read it from USB_CAP3 to USB_CAP5 */
+ ep_enabled_reg = readl(&priv_dev->regs->usb_cap3);
+ iso_ep_reg = readl(&priv_dev->regs->usb_cap4);
+
+ dev_dbg(priv_dev->dev, "Initializing non-zero endpoints\n");
+
+ for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++) {
+ ep_dir = i >> 4; /* i div 16 */
+ ep_number = i & 0xF; /* i % 16 */
+ ep_mask = BIT(i);
+
+ if (!(ep_enabled_reg & ep_mask))
+ continue;
+
+ if (ep_dir && !ep_number) {
+ priv_dev->eps[i] = priv_dev->eps[0];
+ continue;
+ }
+
+ priv_ep = devm_kzalloc(priv_dev->dev, sizeof(*priv_ep),
+ GFP_KERNEL);
+ if (!priv_ep) {
+ ret = -ENOMEM;
+ goto err;
+ }
+
+ /* set parent of endpoint object */
+ priv_ep->cdns3_dev = priv_dev;
+ priv_dev->eps[i] = priv_ep;
+ priv_ep->num = ep_number;
+ priv_ep->dir = ep_dir ? USB_DIR_IN : USB_DIR_OUT;
+
+ if (!ep_number) {
+ ret = cdns3_init_ep0(priv_dev, priv_ep);
+ if (ret) {
+ dev_err(priv_dev->dev, "Failed to init ep0\n");
+ goto err;
+ }
+ } else {
+ snprintf(priv_ep->name, sizeof(priv_ep->name), "ep%d%s",
+ ep_number, !!ep_dir ? "in" : "out");
+ priv_ep->endpoint.name = priv_ep->name;
+
+ usb_ep_set_maxpacket_limit(&priv_ep->endpoint,
+ CDNS3_EP_MAX_PACKET_LIMIT);
+ priv_ep->endpoint.max_streams = CDNS3_EP_MAX_STREAMS;
+ priv_ep->endpoint.ops = &cdns3_gadget_ep_ops;
+ if (ep_dir)
+ priv_ep->endpoint.caps.dir_in = 1;
+ else
+ priv_ep->endpoint.caps.dir_out = 1;
+
+ if (iso_ep_reg & ep_mask)
+ priv_ep->endpoint.caps.type_iso = 1;
+
+ priv_ep->endpoint.caps.type_bulk = 1;
+ priv_ep->endpoint.caps.type_int = 1;
+
+ list_add_tail(&priv_ep->endpoint.ep_list,
+ &priv_dev->gadget.ep_list);
+ }
+
+ priv_ep->flags = 0;
+
+ dev_info(priv_dev->dev, "Initialized %s support: %s %s\n",
+ priv_ep->name,
+ priv_ep->endpoint.caps.type_bulk ? "BULK, INT" : "",
+ priv_ep->endpoint.caps.type_iso ? "ISO" : "");
+
+ INIT_LIST_HEAD(&priv_ep->pending_req_list);
+ INIT_LIST_HEAD(&priv_ep->deferred_req_list);
+ INIT_LIST_HEAD(&priv_ep->wa2_descmiss_req_list);
+ }
+
+ return 0;
+err:
+ cdns3_free_all_eps(priv_dev);
+ return -ENOMEM;
+}
+
+void cdns3_gadget_exit(struct cdns3 *cdns)
+{
+ struct cdns3_device *priv_dev;
+
+ priv_dev = cdns->gadget_dev;
+
+ usb_del_gadget_udc(&priv_dev->gadget);
+
+ cdns3_free_all_eps(priv_dev);
+
+ while (!list_empty(&priv_dev->aligned_buf_list)) {
+ struct cdns3_aligned_buf *buf;
+
+ buf = cdns3_next_align_buf(&priv_dev->aligned_buf_list);
+ dma_free_coherent(buf->buf);
+
+ list_del(&buf->list);
+ kfree(buf);
+ }
+
+ dma_free_coherent(priv_dev->setup_buf);
+
+ kfree(priv_dev->zlp_buf);
+ kfree(priv_dev);
+ cdns->gadget_dev = NULL;
+ cdns3_drd_switch_gadget(cdns, 0);
+}
+
+static int cdns3_gadget_start(struct cdns3 *cdns)
+{
+ struct cdns3_device *priv_dev;
+ u32 max_speed;
+ int ret;
+
+ priv_dev = kzalloc(sizeof(*priv_dev), GFP_KERNEL);
+ if (!priv_dev)
+ return -ENOMEM;
+
+ cdns->gadget_dev = priv_dev;
+ priv_dev->sysdev = cdns->dev;
+ priv_dev->dev = cdns->dev;
+ priv_dev->regs = cdns->dev_regs;
+
+ dev_read_u32(priv_dev->dev, "cdns,on-chip-buff-size",
+ &priv_dev->onchip_buffers);
+
+ if (priv_dev->onchip_buffers <= 0) {
+ u32 reg = readl(&priv_dev->regs->usb_cap2);
+
+ priv_dev->onchip_buffers = USB_CAP2_ACTUAL_MEM_SIZE(reg);
+ }
+
+ if (!priv_dev->onchip_buffers)
+ priv_dev->onchip_buffers = 256;
+
+ max_speed = usb_get_maximum_speed(dev_of_offset(cdns->dev));
+
+ /* Check the maximum_speed parameter */
+ switch (max_speed) {
+ case USB_SPEED_FULL:
+ /* fall through */
+ case USB_SPEED_HIGH:
+ /* fall through */
+ case USB_SPEED_SUPER:
+ break;
+ default:
+ dev_err(cdns->dev, "invalid maximum_speed parameter %d\n",
+ max_speed);
+ /* fall through */
+ case USB_SPEED_UNKNOWN:
+ /* default to superspeed */
+ max_speed = USB_SPEED_SUPER;
+ break;
+ }
+
+ /* fill gadget fields */
+ priv_dev->gadget.max_speed = max_speed;
+ priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
+ priv_dev->gadget.ops = &cdns3_gadget_ops;
+ priv_dev->gadget.name = "cdns3-gadget";
+#ifndef __UBOOT__
+ priv_dev->gadget.name = "usb-ss-gadget";
+ priv_dev->gadget.sg_supported = 1;
+ priv_dev->gadget.quirk_avoids_skb_reserve = 1;
+#endif
+
+ spin_lock_init(&priv_dev->lock);
+ INIT_WORK(&priv_dev->pending_status_wq,
+ cdns3_pending_setup_status_handler);
+
+ /* initialize endpoint container */
+ INIT_LIST_HEAD(&priv_dev->gadget.ep_list);
+ INIT_LIST_HEAD(&priv_dev->aligned_buf_list);
+
+ ret = cdns3_init_eps(priv_dev);
+ if (ret) {
+ dev_err(priv_dev->dev, "Failed to create endpoints\n");
+ goto err1;
+ }
+
+ /* allocate memory for setup packet buffer */
+ priv_dev->setup_buf =
+ dma_alloc_coherent(8, (unsigned long *)&priv_dev->setup_dma);
+ if (!priv_dev->setup_buf) {
+ ret = -ENOMEM;
+ goto err2;
+ }
+
+ priv_dev->dev_ver = readl(&priv_dev->regs->usb_cap6);
+
+ dev_dbg(priv_dev->dev, "Device Controller version: %08x\n",
+ readl(&priv_dev->regs->usb_cap6));
+ dev_dbg(priv_dev->dev, "USB Capabilities:: %08x\n",
+ readl(&priv_dev->regs->usb_cap1));
+ dev_dbg(priv_dev->dev, "On-Chip memory cnfiguration: %08x\n",
+ readl(&priv_dev->regs->usb_cap2));
+
+ priv_dev->dev_ver = GET_DEV_BASE_VERSION(priv_dev->dev_ver);
+
+ priv_dev->zlp_buf = kzalloc(CDNS3_EP_ZLP_BUF_SIZE, GFP_KERNEL);
+ if (!priv_dev->zlp_buf) {
+ ret = -ENOMEM;
+ goto err3;
+ }
+
+ /* add USB gadget device */
+ ret = usb_add_gadget_udc((struct device *)priv_dev->dev,
+ &priv_dev->gadget);
+ if (ret < 0) {
+ dev_err(priv_dev->dev,
+ "Failed to register USB device controller\n");
+ goto err4;
+ }
+
+ return 0;
+err4:
+ kfree(priv_dev->zlp_buf);
+err3:
+ dma_free_coherent(priv_dev->setup_buf);
+err2:
+ cdns3_free_all_eps(priv_dev);
+err1:
+ cdns->gadget_dev = NULL;
+ return ret;
+}
+
+static int __cdns3_gadget_init(struct cdns3 *cdns)
+{
+ int ret = 0;
+
+ cdns3_drd_switch_gadget(cdns, 1);
+
+ ret = cdns3_gadget_start(cdns);
+ if (ret)
+ return ret;
+
+ return 0;
+}
+
+static int cdns3_gadget_suspend(struct cdns3 *cdns, bool do_wakeup)
+{
+ struct cdns3_device *priv_dev = cdns->gadget_dev;
+
+ cdns3_disconnect_gadget(priv_dev);
+
+ priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
+ usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);
+ cdns3_hw_reset_eps_config(priv_dev);
+
+ /* disable interrupt for device */
+ writel(0, &priv_dev->regs->usb_ien);
+
+ cdns3_gadget_pullup(&priv_dev->gadget, 0);
+
+ return 0;
+}
+
+static int cdns3_gadget_resume(struct cdns3 *cdns, bool hibernated)
+{
+ struct cdns3_device *priv_dev = cdns->gadget_dev;
+
+ if (!priv_dev->gadget_driver)
+ return 0;
+
+ cdns3_gadget_config(priv_dev);
+
+ return 0;
+}
+
+/**
+ * cdns3_gadget_init - initialize device structure
+ *
+ * cdns: cdns3 instance
+ *
+ * This function initializes the gadget.
+ */
+int cdns3_gadget_init(struct cdns3 *cdns)
+{
+ struct cdns3_role_driver *rdrv;
+
+ rdrv = devm_kzalloc(cdns->dev, sizeof(*rdrv), GFP_KERNEL);
+ if (!rdrv)
+ return -ENOMEM;
+
+ rdrv->start = __cdns3_gadget_init;
+ rdrv->stop = cdns3_gadget_exit;
+ rdrv->suspend = cdns3_gadget_suspend;
+ rdrv->resume = cdns3_gadget_resume;
+ rdrv->state = CDNS3_ROLE_STATE_INACTIVE;
+ rdrv->name = "gadget";
+ cdns->roles[USB_ROLE_DEVICE] = rdrv;
+
+ return 0;
+}
+
+/**
+ * cdns3_gadget_uboot_handle_interrupt - handle cdns3 gadget interrupt
+ * @cdns: pointer to struct cdns3
+ *
+ * Handles ep0 and gadget interrupt
+ */
+static void cdns3_gadget_uboot_handle_interrupt(struct cdns3 *cdns)
+{
+ int ret = cdns3_device_irq_handler(0, cdns);
+
+ if (ret == IRQ_WAKE_THREAD)
+ cdns3_device_thread_irq_handler(0, cdns);
+}
+
+int dm_usb_gadget_handle_interrupts(struct udevice *dev)
+{
+ struct cdns3 *cdns = dev_get_priv(dev);
+
+ cdns3_gadget_uboot_handle_interrupt(cdns);
+
+ return 0;
+}
projects / platform / kernel / u-boot.git / commitdiff