From: Masahiro Yamada Date: Fri, 14 Feb 2020 07:40:19 +0000 (+0900) Subject: dma-mapping: move dma_map_(un)single() to X-Git-Tag: submit/tizen_6.0/20211217.073345~15 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=b1cce7596fa5972ffcff5375254a8fd108714a90;p=platform%2Fkernel%2Fu-boot.git dma-mapping: move dma_map_(un)single() to The implementation of dma_map_single() and dma_unmap_single() is exactly the same for all the architectures that support them. Factor them out to , and make all drivers to include instead of . If we need to differentiate them for some architectures, we can move the generic definitions to . Add some comments to the helpers. The concept is quite similar to the DMA-API of Linux kernel. Drivers are agnostic about what is going on behind the scene. Just call dma_map_single() before the DMA, and dma_unmap_single() after it. Change-Id: I190e0ab3cdb35d56b918dba41151344d4b97402c Signed-off-by: Masahiro Yamada --- diff --git a/arch/arm/include/asm/dma-mapping.h b/arch/arm/include/asm/dma-mapping.h index 895b24701f..284fac79f5 100644 --- a/arch/arm/include/asm/dma-mapping.h +++ b/arch/arm/include/asm/dma-mapping.h @@ -9,8 +9,6 @@ #include -#define dma_mapping_error(x, y) 0 - static inline void *dma_alloc_coherent(size_t len, unsigned long *handle) { *handle = (unsigned long)memalign(ARCH_DMA_MINALIGN, ROUND(len, ARCH_DMA_MINALIGN)); @@ -22,19 +20,4 @@ static inline void dma_free_coherent(void *addr) free(addr); } -static inline unsigned long dma_map_single(volatile void *vaddr, size_t len, - enum dma_data_direction dir) -{ - return (unsigned long)vaddr; -} - -static inline void dma_unmap_single(dma_addr_t addr, size_t len, - enum dma_data_direction dir) -{ - len = ALIGN(len, ARCH_DMA_MINALIGN); - - if (dir != DMA_TO_DEVICE) - invalidate_dcache_range(addr, addr + len); -} - #endif /* __ASM_ARM_DMA_MAPPING_H */ diff --git a/arch/nds32/include/asm/dma-mapping.h b/arch/nds32/include/asm/dma-mapping.h index ae6f1ad42f..4f74eac3ef 100644 --- a/arch/nds32/include/asm/dma-mapping.h +++ b/arch/nds32/include/asm/dma-mapping.h @@ -14,19 +14,4 @@ static void *dma_alloc_coherent(size_t len, unsigned long *handle) return (void *)*handle; } -static inline unsigned long dma_map_single(volatile void *vaddr, size_t len, - enum dma_data_direction dir) -{ - return (unsigned long)vaddr; -} - -static inline void dma_unmap_single(dma_addr_t addr, size_t len, - enum dma_data_direction dir) -{ - len = ALIGN(len, ARCH_DMA_MINALIGN); - - if (dir != DMA_TO_DEVICE) - invalidate_dcache_range(addr, addr + len); -} - #endif /* __ASM_NDS_DMA_MAPPING_H */ diff --git a/arch/riscv/include/asm/dma-mapping.h b/arch/riscv/include/asm/dma-mapping.h index 8bbf0f9848..4e033c5797 100644 --- a/arch/riscv/include/asm/dma-mapping.h +++ b/arch/riscv/include/asm/dma-mapping.h @@ -11,8 +11,6 @@ #include -#define dma_mapping_error(x, y) 0 - static inline void *dma_alloc_coherent(size_t len, unsigned long *handle) { *handle = (unsigned long)memalign(ARCH_DMA_MINALIGN, len); @@ -24,19 +22,4 @@ static inline void dma_free_coherent(void *addr) free(addr); } -static inline unsigned long dma_map_single(volatile void *vaddr, size_t len, - enum dma_data_direction dir) -{ - return (unsigned long)vaddr; -} - -static inline void dma_unmap_single(dma_addr_t addr, size_t len, - enum dma_data_direction dir) -{ - len = ALIGN(len, ARCH_DMA_MINALIGN); - - if (dir != DMA_TO_DEVICE) - invalidate_dcache_range(addr, addr + len); -} - #endif /* __ASM_RISCV_DMA_MAPPING_H */ diff --git a/arch/x86/include/asm/dma-mapping.h b/arch/x86/include/asm/dma-mapping.h index d83cf8ec5b..1317f9a7b3 100644 --- a/arch/x86/include/asm/dma-mapping.h +++ b/arch/x86/include/asm/dma-mapping.h @@ -9,8 +9,6 @@ #include -#define dma_mapping_error(x, y) 0 - static inline void *dma_alloc_coherent(size_t len, unsigned long *handle) { *handle = (unsigned long)memalign(ARCH_DMA_MINALIGN, len); @@ -22,19 +20,4 @@ static inline void dma_free_coherent(void *addr) free(addr); } -static inline unsigned long dma_map_single(volatile void *vaddr, size_t len, - enum dma_data_direction dir) -{ - return (unsigned long)vaddr; -} - -static inline void dma_unmap_single(dma_addr_t addr, size_t len, - enum dma_data_direction dir) -{ - len = ALIGN(len, ARCH_DMA_MINALIGN); - - if (dir != DMA_TO_DEVICE) - invalidate_dcache_range(addr, addr + len); -} - #endif /* __ASM_X86_DMA_MAPPING_H */ diff --git a/drivers/dma/ti/k3-udma.c b/drivers/dma/ti/k3-udma.c index a5fc7809bc..fa8b26e380 100644 --- a/drivers/dma/ti/k3-udma.c +++ b/drivers/dma/ti/k3-udma.c @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/mmc/tmio-common.c b/drivers/mmc/tmio-common.c index e25b95f424..a91edab86d 100644 --- a/drivers/mmc/tmio-common.c +++ b/drivers/mmc/tmio-common.c @@ -11,7 +11,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/drivers/mtd/nand/raw/denali.c b/drivers/mtd/nand/raw/denali.c index 78204da7ca..4ba7ffbfc6 100644 --- a/drivers/mtd/nand/raw/denali.c +++ b/drivers/mtd/nand/raw/denali.c @@ -8,7 +8,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/net/altera_tse.c b/drivers/net/altera_tse.c index fb878d4e63..95d39581af 100644 --- a/drivers/net/altera_tse.c +++ b/drivers/net/altera_tse.c @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include #include "altera_tse.h" diff --git a/drivers/net/ftmac110.c b/drivers/net/ftmac110.c index 1fa93d4d58..84082407d2 100644 --- a/drivers/net/ftmac110.c +++ b/drivers/net/ftmac110.c @@ -12,7 +12,7 @@ #include #include #include -#include +#include #if defined(CONFIG_MII) || defined(CONFIG_CMD_MII) #include diff --git a/drivers/net/macb.c b/drivers/net/macb.c index 245a546cae..d4637b338e 100644 --- a/drivers/net/macb.c +++ b/drivers/net/macb.c @@ -37,7 +37,7 @@ #include #include -#include +#include #include #include diff --git a/drivers/soc/ti/k3-navss-ringacc.c b/drivers/soc/ti/k3-navss-ringacc.c index 64ebc0ba00..cabdce7c2e 100644 --- a/drivers/soc/ti/k3-navss-ringacc.c +++ b/drivers/soc/ti/k3-navss-ringacc.c @@ -8,12 +8,13 @@ #include #include #include -#include #include #include #include #include #include +#include +#include #include #include diff --git a/drivers/ufs/ufs.c b/drivers/ufs/ufs.c new file mode 100644 index 0000000000..24e1bc2a19 --- /dev/null +++ b/drivers/ufs/ufs.c @@ -0,0 +1,1970 @@ +// SPDX-License-Identifier: GPL-2.0+ +/** + * ufs.c - Universal Flash Subsystem (UFS) driver + * + * Taken from Linux Kernel v5.2 (drivers/scsi/ufs/ufshcd.c) and ported + * to u-boot. + * + * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "ufs.h" + +#define UFSHCD_ENABLE_INTRS (UTP_TRANSFER_REQ_COMPL |\ + UTP_TASK_REQ_COMPL |\ + UFSHCD_ERROR_MASK) +/* maximum number of link-startup retries */ +#define DME_LINKSTARTUP_RETRIES 3 + +/* maximum number of retries for a general UIC command */ +#define UFS_UIC_COMMAND_RETRIES 3 + +/* Query request retries */ +#define QUERY_REQ_RETRIES 3 +/* Query request timeout */ +#define QUERY_REQ_TIMEOUT 1500 /* 1.5 seconds */ + +/* maximum timeout in ms for a general UIC command */ +#define UFS_UIC_CMD_TIMEOUT 1000 +/* NOP OUT retries waiting for NOP IN response */ +#define NOP_OUT_RETRIES 10 +/* Timeout after 30 msecs if NOP OUT hangs without response */ +#define NOP_OUT_TIMEOUT 30 /* msecs */ + +/* Only use one Task Tag for all requests */ +#define TASK_TAG 0 + +/* Expose the flag value from utp_upiu_query.value */ +#define MASK_QUERY_UPIU_FLAG_LOC 0xFF + +#define MAX_PRDT_ENTRY 262144 + +/* maximum bytes per request */ +#define UFS_MAX_BYTES (128 * 256 * 1024) + +static inline bool ufshcd_is_hba_active(struct ufs_hba *hba); +static inline void ufshcd_hba_stop(struct ufs_hba *hba); +static int ufshcd_hba_enable(struct ufs_hba *hba); + +/* + * ufshcd_wait_for_register - wait for register value to change + */ +static int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask, + u32 val, unsigned long timeout_ms) +{ + int err = 0; + unsigned long start = get_timer(0); + + /* ignore bits that we don't intend to wait on */ + val = val & mask; + + while ((ufshcd_readl(hba, reg) & mask) != val) { + if (get_timer(start) > timeout_ms) { + if ((ufshcd_readl(hba, reg) & mask) != val) + err = -ETIMEDOUT; + break; + } + } + + return err; +} + +/** + * ufshcd_init_pwr_info - setting the POR (power on reset) + * values in hba power info + */ +static void ufshcd_init_pwr_info(struct ufs_hba *hba) +{ + hba->pwr_info.gear_rx = UFS_PWM_G1; + hba->pwr_info.gear_tx = UFS_PWM_G1; + hba->pwr_info.lane_rx = 1; + hba->pwr_info.lane_tx = 1; + hba->pwr_info.pwr_rx = SLOWAUTO_MODE; + hba->pwr_info.pwr_tx = SLOWAUTO_MODE; + hba->pwr_info.hs_rate = 0; +} + +/** + * ufshcd_print_pwr_info - print power params as saved in hba + * power info + */ +static void ufshcd_print_pwr_info(struct ufs_hba *hba) +{ + static const char * const names[] = { + "INVALID MODE", + "FAST MODE", + "SLOW_MODE", + "INVALID MODE", + "FASTAUTO_MODE", + "SLOWAUTO_MODE", + "INVALID MODE", + }; + + dev_err(hba->dev, "[RX, TX]: gear=[%d, %d], lane[%d, %d], pwr[%s, %s], rate = %d\n", + hba->pwr_info.gear_rx, hba->pwr_info.gear_tx, + hba->pwr_info.lane_rx, hba->pwr_info.lane_tx, + names[hba->pwr_info.pwr_rx], + names[hba->pwr_info.pwr_tx], + hba->pwr_info.hs_rate); +} + +/** + * ufshcd_ready_for_uic_cmd - Check if controller is ready + * to accept UIC commands + */ +static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba) +{ + if (ufshcd_readl(hba, REG_CONTROLLER_STATUS) & UIC_COMMAND_READY) + return true; + else + return false; +} + +/** + * ufshcd_get_uic_cmd_result - Get the UIC command result + */ +static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba) +{ + return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2) & + MASK_UIC_COMMAND_RESULT; +} + +/** + * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command + */ +static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba) +{ + return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3); +} + +/** + * ufshcd_is_device_present - Check if any device connected to + * the host controller + */ +static inline bool ufshcd_is_device_present(struct ufs_hba *hba) +{ + return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) & + DEVICE_PRESENT) ? true : false; +} + +/** + * ufshcd_send_uic_cmd - UFS Interconnect layer command API + * + */ +static int ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd) +{ + unsigned long start = 0; + u32 intr_status; + u32 enabled_intr_status; + + if (!ufshcd_ready_for_uic_cmd(hba)) { + dev_err(hba->dev, + "Controller not ready to accept UIC commands\n"); + return -EIO; + } + + debug("sending uic command:%d\n", uic_cmd->command); + + /* Write Args */ + ufshcd_writel(hba, uic_cmd->argument1, REG_UIC_COMMAND_ARG_1); + ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2); + ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3); + + /* Write UIC Cmd */ + ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK, + REG_UIC_COMMAND); + + start = get_timer(0); + do { + intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS); + enabled_intr_status = intr_status & hba->intr_mask; + ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS); + + if (get_timer(start) > UFS_UIC_CMD_TIMEOUT) { + dev_err(hba->dev, + "Timedout waiting for UIC response\n"); + + return -ETIMEDOUT; + } + + if (enabled_intr_status & UFSHCD_ERROR_MASK) { + dev_err(hba->dev, "Error in status:%08x\n", + enabled_intr_status); + + return -1; + } + } while (!(enabled_intr_status & UFSHCD_UIC_MASK)); + + uic_cmd->argument2 = ufshcd_get_uic_cmd_result(hba); + uic_cmd->argument3 = ufshcd_get_dme_attr_val(hba); + + debug("Sent successfully\n"); + + return 0; +} + +/** + * ufshcd_dme_set_attr - UIC command for DME_SET, DME_PEER_SET + * + */ +int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel, u8 attr_set, + u32 mib_val, u8 peer) +{ + struct uic_command uic_cmd = {0}; + static const char *const action[] = { + "dme-set", + "dme-peer-set" + }; + const char *set = action[!!peer]; + int ret; + int retries = UFS_UIC_COMMAND_RETRIES; + + uic_cmd.command = peer ? + UIC_CMD_DME_PEER_SET : UIC_CMD_DME_SET; + uic_cmd.argument1 = attr_sel; + uic_cmd.argument2 = UIC_ARG_ATTR_TYPE(attr_set); + uic_cmd.argument3 = mib_val; + + do { + /* for peer attributes we retry upon failure */ + ret = ufshcd_send_uic_cmd(hba, &uic_cmd); + if (ret) + dev_dbg(hba->dev, "%s: attr-id 0x%x val 0x%x error code %d\n", + set, UIC_GET_ATTR_ID(attr_sel), mib_val, ret); + } while (ret && peer && --retries); + + if (ret) + dev_err(hba->dev, "%s: attr-id 0x%x val 0x%x failed %d retries\n", + set, UIC_GET_ATTR_ID(attr_sel), mib_val, + UFS_UIC_COMMAND_RETRIES - retries); + + return ret; +} + +/** + * ufshcd_dme_get_attr - UIC command for DME_GET, DME_PEER_GET + * + */ +int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel, + u32 *mib_val, u8 peer) +{ + struct uic_command uic_cmd = {0}; + static const char *const action[] = { + "dme-get", + "dme-peer-get" + }; + const char *get = action[!!peer]; + int ret; + int retries = UFS_UIC_COMMAND_RETRIES; + + uic_cmd.command = peer ? + UIC_CMD_DME_PEER_GET : UIC_CMD_DME_GET; + uic_cmd.argument1 = attr_sel; + + do { + /* for peer attributes we retry upon failure */ + ret = ufshcd_send_uic_cmd(hba, &uic_cmd); + if (ret) + dev_dbg(hba->dev, "%s: attr-id 0x%x error code %d\n", + get, UIC_GET_ATTR_ID(attr_sel), ret); + } while (ret && peer && --retries); + + if (ret) + dev_err(hba->dev, "%s: attr-id 0x%x failed %d retries\n", + get, UIC_GET_ATTR_ID(attr_sel), + UFS_UIC_COMMAND_RETRIES - retries); + + if (mib_val && !ret) + *mib_val = uic_cmd.argument3; + + return ret; +} + +static int ufshcd_disable_tx_lcc(struct ufs_hba *hba, bool peer) +{ + u32 tx_lanes, i, err = 0; + + if (!peer) + ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES), + &tx_lanes); + else + ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES), + &tx_lanes); + for (i = 0; i < tx_lanes; i++) { + if (!peer) + err = ufshcd_dme_set(hba, + UIC_ARG_MIB_SEL(TX_LCC_ENABLE, + UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)), + 0); + else + err = ufshcd_dme_peer_set(hba, + UIC_ARG_MIB_SEL(TX_LCC_ENABLE, + UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)), + 0); + if (err) { + dev_err(hba->dev, "%s: TX LCC Disable failed, peer = %d, lane = %d, err = %d", + __func__, peer, i, err); + break; + } + } + + return err; +} + +static inline int ufshcd_disable_device_tx_lcc(struct ufs_hba *hba) +{ + return ufshcd_disable_tx_lcc(hba, true); +} + +/** + * ufshcd_dme_link_startup - Notify Unipro to perform link startup + * + */ +static int ufshcd_dme_link_startup(struct ufs_hba *hba) +{ + struct uic_command uic_cmd = {0}; + int ret; + + uic_cmd.command = UIC_CMD_DME_LINK_STARTUP; + + ret = ufshcd_send_uic_cmd(hba, &uic_cmd); + if (ret) + dev_dbg(hba->dev, + "dme-link-startup: error code %d\n", ret); + return ret; +} + +/** + * ufshcd_disable_intr_aggr - Disables interrupt aggregation. + * + */ +static inline void ufshcd_disable_intr_aggr(struct ufs_hba *hba) +{ + ufshcd_writel(hba, 0, REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL); +} + +/** + * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY + */ +static inline int ufshcd_get_lists_status(u32 reg) +{ + return !((reg & UFSHCD_STATUS_READY) == UFSHCD_STATUS_READY); +} + +/** + * ufshcd_enable_run_stop_reg - Enable run-stop registers, + * When run-stop registers are set to 1, it indicates the + * host controller that it can process the requests + */ +static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba) +{ + ufshcd_writel(hba, UTP_TASK_REQ_LIST_RUN_STOP_BIT, + REG_UTP_TASK_REQ_LIST_RUN_STOP); + ufshcd_writel(hba, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT, + REG_UTP_TRANSFER_REQ_LIST_RUN_STOP); +} + +/** + * ufshcd_enable_intr - enable interrupts + */ +static void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs) +{ + u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE); + u32 rw; + + if (hba->version == UFSHCI_VERSION_10) { + rw = set & INTERRUPT_MASK_RW_VER_10; + set = rw | ((set ^ intrs) & intrs); + } else { + set |= intrs; + } + + ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE); + + hba->intr_mask = set; +} + +/** + * ufshcd_make_hba_operational - Make UFS controller operational + * + * To bring UFS host controller to operational state, + * 1. Enable required interrupts + * 2. Configure interrupt aggregation + * 3. Program UTRL and UTMRL base address + * 4. Configure run-stop-registers + * + */ +static int ufshcd_make_hba_operational(struct ufs_hba *hba) +{ + int err = 0; + u32 reg; + + /* Enable required interrupts */ + ufshcd_enable_intr(hba, UFSHCD_ENABLE_INTRS); + + /* Disable interrupt aggregation */ + ufshcd_disable_intr_aggr(hba); + + /* Configure UTRL and UTMRL base address registers */ + ufshcd_writel(hba, lower_32_bits((dma_addr_t)hba->utrdl), + REG_UTP_TRANSFER_REQ_LIST_BASE_L); + ufshcd_writel(hba, upper_32_bits((dma_addr_t)hba->utrdl), + REG_UTP_TRANSFER_REQ_LIST_BASE_H); + ufshcd_writel(hba, lower_32_bits((dma_addr_t)hba->utmrdl), + REG_UTP_TASK_REQ_LIST_BASE_L); + ufshcd_writel(hba, upper_32_bits((dma_addr_t)hba->utmrdl), + REG_UTP_TASK_REQ_LIST_BASE_H); + + /* + * UCRDY, UTMRLDY and UTRLRDY bits must be 1 + */ + reg = ufshcd_readl(hba, REG_CONTROLLER_STATUS); + if (!(ufshcd_get_lists_status(reg))) { + ufshcd_enable_run_stop_reg(hba); + } else { + dev_err(hba->dev, + "Host controller not ready to process requests"); + err = -EIO; + goto out; + } + +out: + return err; +} + +/** + * ufshcd_link_startup - Initialize unipro link startup + */ +static int ufshcd_link_startup(struct ufs_hba *hba) +{ + int ret; + int retries = DME_LINKSTARTUP_RETRIES; + bool link_startup_again = true; + +link_startup: + do { + ufshcd_ops_link_startup_notify(hba, PRE_CHANGE); + + ret = ufshcd_dme_link_startup(hba); + + /* check if device is detected by inter-connect layer */ + if (!ret && !ufshcd_is_device_present(hba)) { + dev_err(hba->dev, "%s: Device not present\n", __func__); + ret = -ENXIO; + goto out; + } + + /* + * DME link lost indication is only received when link is up, + * but we can't be sure if the link is up until link startup + * succeeds. So reset the local Uni-Pro and try again. + */ + if (ret && ufshcd_hba_enable(hba)) + goto out; + } while (ret && retries--); + + if (ret) + /* failed to get the link up... retire */ + goto out; + + if (link_startup_again) { + link_startup_again = false; + retries = DME_LINKSTARTUP_RETRIES; + goto link_startup; + } + + /* Mark that link is up in PWM-G1, 1-lane, SLOW-AUTO mode */ + ufshcd_init_pwr_info(hba); + + if (hba->quirks & UFSHCD_QUIRK_BROKEN_LCC) { + ret = ufshcd_disable_device_tx_lcc(hba); + if (ret) + goto out; + } + + /* Include any host controller configuration via UIC commands */ + ret = ufshcd_ops_link_startup_notify(hba, POST_CHANGE); + if (ret) + goto out; + + ret = ufshcd_make_hba_operational(hba); +out: + if (ret) + dev_err(hba->dev, "link startup failed %d\n", ret); + + return ret; +} + +/** + * ufshcd_hba_stop - Send controller to reset state + */ +static inline void ufshcd_hba_stop(struct ufs_hba *hba) +{ + int err; + + ufshcd_writel(hba, CONTROLLER_DISABLE, REG_CONTROLLER_ENABLE); + err = ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE, + CONTROLLER_ENABLE, CONTROLLER_DISABLE, + 10); + if (err) + dev_err(hba->dev, "%s: Controller disable failed\n", __func__); +} + +/** + * ufshcd_is_hba_active - Get controller state + */ +static inline bool ufshcd_is_hba_active(struct ufs_hba *hba) +{ + return (ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & CONTROLLER_ENABLE) + ? false : true; +} + +/** + * ufshcd_hba_start - Start controller initialization sequence + */ +static inline void ufshcd_hba_start(struct ufs_hba *hba) +{ + ufshcd_writel(hba, CONTROLLER_ENABLE, REG_CONTROLLER_ENABLE); +} + +/** + * ufshcd_hba_enable - initialize the controller + */ +static int ufshcd_hba_enable(struct ufs_hba *hba) +{ + int retry; + + if (!ufshcd_is_hba_active(hba)) + /* change controller state to "reset state" */ + ufshcd_hba_stop(hba); + + ufshcd_ops_hce_enable_notify(hba, PRE_CHANGE); + + /* start controller initialization sequence */ + ufshcd_hba_start(hba); + + /* + * To initialize a UFS host controller HCE bit must be set to 1. + * During initialization the HCE bit value changes from 1->0->1. + * When the host controller completes initialization sequence + * it sets the value of HCE bit to 1. The same HCE bit is read back + * to check if the controller has completed initialization sequence. + * So without this delay the value HCE = 1, set in the previous + * instruction might be read back. + * This delay can be changed based on the controller. + */ + mdelay(1); + + /* wait for the host controller to complete initialization */ + retry = 10; + while (ufshcd_is_hba_active(hba)) { + if (retry) { + retry--; + } else { + dev_err(hba->dev, "Controller enable failed\n"); + return -EIO; + } + mdelay(5); + } + + /* enable UIC related interrupts */ + ufshcd_enable_intr(hba, UFSHCD_UIC_MASK); + + ufshcd_ops_hce_enable_notify(hba, POST_CHANGE); + + return 0; +} + +/** + * ufshcd_host_memory_configure - configure local reference block with + * memory offsets + */ +static void ufshcd_host_memory_configure(struct ufs_hba *hba) +{ + struct utp_transfer_req_desc *utrdlp; + dma_addr_t cmd_desc_dma_addr; + u16 response_offset; + u16 prdt_offset; + + utrdlp = hba->utrdl; + cmd_desc_dma_addr = (dma_addr_t)hba->ucdl; + + utrdlp->command_desc_base_addr_lo = + cpu_to_le32(lower_32_bits(cmd_desc_dma_addr)); + utrdlp->command_desc_base_addr_hi = + cpu_to_le32(upper_32_bits(cmd_desc_dma_addr)); + + response_offset = offsetof(struct utp_transfer_cmd_desc, response_upiu); + prdt_offset = offsetof(struct utp_transfer_cmd_desc, prd_table); + + utrdlp->response_upiu_offset = cpu_to_le16(response_offset >> 2); + utrdlp->prd_table_offset = cpu_to_le16(prdt_offset >> 2); + utrdlp->response_upiu_length = cpu_to_le16(ALIGNED_UPIU_SIZE >> 2); + + hba->ucd_req_ptr = (struct utp_upiu_req *)hba->ucdl; + hba->ucd_rsp_ptr = + (struct utp_upiu_rsp *)&hba->ucdl->response_upiu; + hba->ucd_prdt_ptr = + (struct ufshcd_sg_entry *)&hba->ucdl->prd_table; +} + +/** + * ufshcd_memory_alloc - allocate memory for host memory space data structures + */ +static int ufshcd_memory_alloc(struct ufs_hba *hba) +{ + /* Allocate one Transfer Request Descriptor + * Should be aligned to 1k boundary. + */ + hba->utrdl = memalign(1024, sizeof(struct utp_transfer_req_desc)); + if (!hba->utrdl) { + dev_err(hba->dev, "Transfer Descriptor memory allocation failed\n"); + return -ENOMEM; + } + + /* Allocate one Command Descriptor + * Should be aligned to 1k boundary. + */ + hba->ucdl = memalign(1024, sizeof(struct utp_transfer_cmd_desc)); + if (!hba->ucdl) { + dev_err(hba->dev, "Command descriptor memory allocation failed\n"); + return -ENOMEM; + } + + return 0; +} + +/** + * ufshcd_get_intr_mask - Get the interrupt bit mask + */ +static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba) +{ + u32 intr_mask = 0; + + switch (hba->version) { + case UFSHCI_VERSION_10: + intr_mask = INTERRUPT_MASK_ALL_VER_10; + break; + case UFSHCI_VERSION_11: + case UFSHCI_VERSION_20: + intr_mask = INTERRUPT_MASK_ALL_VER_11; + break; + case UFSHCI_VERSION_21: + default: + intr_mask = INTERRUPT_MASK_ALL_VER_21; + break; + } + + return intr_mask; +} + +/** + * ufshcd_get_ufs_version - Get the UFS version supported by the HBA + */ +static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba) +{ + return ufshcd_readl(hba, REG_UFS_VERSION); +} + +/** + * ufshcd_get_upmcrs - Get the power mode change request status + */ +static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba) +{ + return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) >> 8) & 0x7; +} + +/** + * ufshcd_prepare_req_desc_hdr() - Fills the requests header + * descriptor according to request + */ +static void ufshcd_prepare_req_desc_hdr(struct utp_transfer_req_desc *req_desc, + u32 *upiu_flags, + enum dma_data_direction cmd_dir) +{ + u32 data_direction; + u32 dword_0; + + if (cmd_dir == DMA_FROM_DEVICE) { + data_direction = UTP_DEVICE_TO_HOST; + *upiu_flags = UPIU_CMD_FLAGS_READ; + } else if (cmd_dir == DMA_TO_DEVICE) { + data_direction = UTP_HOST_TO_DEVICE; + *upiu_flags = UPIU_CMD_FLAGS_WRITE; + } else { + data_direction = UTP_NO_DATA_TRANSFER; + *upiu_flags = UPIU_CMD_FLAGS_NONE; + } + + dword_0 = data_direction | (0x1 << UPIU_COMMAND_TYPE_OFFSET); + + /* Enable Interrupt for command */ + dword_0 |= UTP_REQ_DESC_INT_CMD; + + /* Transfer request descriptor header fields */ + req_desc->header.dword_0 = cpu_to_le32(dword_0); + /* dword_1 is reserved, hence it is set to 0 */ + req_desc->header.dword_1 = 0; + /* + * assigning invalid value for command status. Controller + * updates OCS on command completion, with the command + * status + */ + req_desc->header.dword_2 = + cpu_to_le32(OCS_INVALID_COMMAND_STATUS); + /* dword_3 is reserved, hence it is set to 0 */ + req_desc->header.dword_3 = 0; + + req_desc->prd_table_length = 0; +} + +static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba, + u32 upiu_flags) +{ + struct utp_upiu_req *ucd_req_ptr = hba->ucd_req_ptr; + struct ufs_query *query = &hba->dev_cmd.query; + u16 len = be16_to_cpu(query->request.upiu_req.length); + + /* Query request header */ + ucd_req_ptr->header.dword_0 = + UPIU_HEADER_DWORD(UPIU_TRANSACTION_QUERY_REQ, + upiu_flags, 0, TASK_TAG); + ucd_req_ptr->header.dword_1 = + UPIU_HEADER_DWORD(0, query->request.query_func, + 0, 0); + + /* Data segment length only need for WRITE_DESC */ + if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC) + ucd_req_ptr->header.dword_2 = + UPIU_HEADER_DWORD(0, 0, (len >> 8), (u8)len); + else + ucd_req_ptr->header.dword_2 = 0; + + /* Copy the Query Request buffer as is */ + memcpy(&ucd_req_ptr->qr, &query->request.upiu_req, QUERY_OSF_SIZE); + + /* Copy the Descriptor */ + if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC) + memcpy(ucd_req_ptr + 1, query->descriptor, len); + + memset(hba->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); +} + +static inline void ufshcd_prepare_utp_nop_upiu(struct ufs_hba *hba) +{ + struct utp_upiu_req *ucd_req_ptr = hba->ucd_req_ptr; + + memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req)); + + /* command descriptor fields */ + ucd_req_ptr->header.dword_0 = + UPIU_HEADER_DWORD(UPIU_TRANSACTION_NOP_OUT, 0, 0, 0x1f); + /* clear rest of the fields of basic header */ + ucd_req_ptr->header.dword_1 = 0; + ucd_req_ptr->header.dword_2 = 0; + + memset(hba->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); +} + +/** + * ufshcd_comp_devman_upiu - UFS Protocol Information Unit(UPIU) + * for Device Management Purposes + */ +static int ufshcd_comp_devman_upiu(struct ufs_hba *hba, + enum dev_cmd_type cmd_type) +{ + u32 upiu_flags; + int ret = 0; + struct utp_transfer_req_desc *req_desc = hba->utrdl; + + hba->dev_cmd.type = cmd_type; + + ufshcd_prepare_req_desc_hdr(req_desc, &upiu_flags, DMA_NONE); + switch (cmd_type) { + case DEV_CMD_TYPE_QUERY: + ufshcd_prepare_utp_query_req_upiu(hba, upiu_flags); + break; + case DEV_CMD_TYPE_NOP: + ufshcd_prepare_utp_nop_upiu(hba); + break; + default: + ret = -EINVAL; + } + + return ret; +} + +static int ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag) +{ + unsigned long start; + u32 intr_status; + u32 enabled_intr_status; + + ufshcd_writel(hba, 1 << task_tag, REG_UTP_TRANSFER_REQ_DOOR_BELL); + + start = get_timer(0); + do { + intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS); + enabled_intr_status = intr_status & hba->intr_mask; + ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS); + + if (get_timer(start) > QUERY_REQ_TIMEOUT) { + dev_err(hba->dev, + "Timedout waiting for UTP response\n"); + + return -ETIMEDOUT; + } + + if (enabled_intr_status & UFSHCD_ERROR_MASK) { + dev_err(hba->dev, "Error in status:%08x\n", + enabled_intr_status); + + return -1; + } + } while (!(enabled_intr_status & UTP_TRANSFER_REQ_COMPL)); + + return 0; +} + +/** + * ufshcd_get_req_rsp - returns the TR response transaction type + */ +static inline int ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr) +{ + return be32_to_cpu(ucd_rsp_ptr->header.dword_0) >> 24; +} + +/** + * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status + * + */ +static inline int ufshcd_get_tr_ocs(struct ufs_hba *hba) +{ + return le32_to_cpu(hba->utrdl->header.dword_2) & MASK_OCS; +} + +static inline int ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp *ucd_rsp_ptr) +{ + return be32_to_cpu(ucd_rsp_ptr->header.dword_1) & MASK_RSP_UPIU_RESULT; +} + +static int ufshcd_check_query_response(struct ufs_hba *hba) +{ + struct ufs_query_res *query_res = &hba->dev_cmd.query.response; + + /* Get the UPIU response */ + query_res->response = ufshcd_get_rsp_upiu_result(hba->ucd_rsp_ptr) >> + UPIU_RSP_CODE_OFFSET; + return query_res->response; +} + +/** + * ufshcd_copy_query_response() - Copy the Query Response and the data + * descriptor + */ +static int ufshcd_copy_query_response(struct ufs_hba *hba) +{ + struct ufs_query_res *query_res = &hba->dev_cmd.query.response; + + memcpy(&query_res->upiu_res, &hba->ucd_rsp_ptr->qr, QUERY_OSF_SIZE); + + /* Get the descriptor */ + if (hba->dev_cmd.query.descriptor && + hba->ucd_rsp_ptr->qr.opcode == UPIU_QUERY_OPCODE_READ_DESC) { + u8 *descp = (u8 *)hba->ucd_rsp_ptr + + GENERAL_UPIU_REQUEST_SIZE; + u16 resp_len; + u16 buf_len; + + /* data segment length */ + resp_len = be32_to_cpu(hba->ucd_rsp_ptr->header.dword_2) & + MASK_QUERY_DATA_SEG_LEN; + buf_len = + be16_to_cpu(hba->dev_cmd.query.request.upiu_req.length); + if (likely(buf_len >= resp_len)) { + memcpy(hba->dev_cmd.query.descriptor, descp, resp_len); + } else { + dev_warn(hba->dev, + "%s: Response size is bigger than buffer", + __func__); + return -EINVAL; + } + } + + return 0; +} + +/** + * ufshcd_exec_dev_cmd - API for sending device management requests + */ +static int ufshcd_exec_dev_cmd(struct ufs_hba *hba, enum dev_cmd_type cmd_type, + int timeout) +{ + int err; + int resp; + + err = ufshcd_comp_devman_upiu(hba, cmd_type); + if (err) + return err; + + err = ufshcd_send_command(hba, TASK_TAG); + if (err) + return err; + + err = ufshcd_get_tr_ocs(hba); + if (err) { + dev_err(hba->dev, "Error in OCS:%d\n", err); + return -EINVAL; + } + + resp = ufshcd_get_req_rsp(hba->ucd_rsp_ptr); + switch (resp) { + case UPIU_TRANSACTION_NOP_IN: + break; + case UPIU_TRANSACTION_QUERY_RSP: + err = ufshcd_check_query_response(hba); + if (!err) + err = ufshcd_copy_query_response(hba); + break; + case UPIU_TRANSACTION_REJECT_UPIU: + /* TODO: handle Reject UPIU Response */ + err = -EPERM; + dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n", + __func__); + break; + default: + err = -EINVAL; + dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n", + __func__, resp); + } + + return err; +} + +/** + * ufshcd_init_query() - init the query response and request parameters + */ +static inline void ufshcd_init_query(struct ufs_hba *hba, + struct ufs_query_req **request, + struct ufs_query_res **response, + enum query_opcode opcode, + u8 idn, u8 index, u8 selector) +{ + *request = &hba->dev_cmd.query.request; + *response = &hba->dev_cmd.query.response; + memset(*request, 0, sizeof(struct ufs_query_req)); + memset(*response, 0, sizeof(struct ufs_query_res)); + (*request)->upiu_req.opcode = opcode; + (*request)->upiu_req.idn = idn; + (*request)->upiu_req.index = index; + (*request)->upiu_req.selector = selector; +} + +/** + * ufshcd_query_flag() - API function for sending flag query requests + */ +int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode, + enum flag_idn idn, bool *flag_res) +{ + struct ufs_query_req *request = NULL; + struct ufs_query_res *response = NULL; + int err, index = 0, selector = 0; + int timeout = QUERY_REQ_TIMEOUT; + + ufshcd_init_query(hba, &request, &response, opcode, idn, index, + selector); + + switch (opcode) { + case UPIU_QUERY_OPCODE_SET_FLAG: + case UPIU_QUERY_OPCODE_CLEAR_FLAG: + case UPIU_QUERY_OPCODE_TOGGLE_FLAG: + request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST; + break; + case UPIU_QUERY_OPCODE_READ_FLAG: + request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST; + if (!flag_res) { + /* No dummy reads */ + dev_err(hba->dev, "%s: Invalid argument for read request\n", + __func__); + err = -EINVAL; + goto out; + } + break; + default: + dev_err(hba->dev, + "%s: Expected query flag opcode but got = %d\n", + __func__, opcode); + err = -EINVAL; + goto out; + } + + err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, timeout); + + if (err) { + dev_err(hba->dev, + "%s: Sending flag query for idn %d failed, err = %d\n", + __func__, idn, err); + goto out; + } + + if (flag_res) + *flag_res = (be32_to_cpu(response->upiu_res.value) & + MASK_QUERY_UPIU_FLAG_LOC) & 0x1; + +out: + return err; +} + +static int ufshcd_query_flag_retry(struct ufs_hba *hba, + enum query_opcode opcode, + enum flag_idn idn, bool *flag_res) +{ + int ret; + int retries; + + for (retries = 0; retries < QUERY_REQ_RETRIES; retries++) { + ret = ufshcd_query_flag(hba, opcode, idn, flag_res); + if (ret) + dev_dbg(hba->dev, + "%s: failed with error %d, retries %d\n", + __func__, ret, retries); + else + break; + } + + if (ret) + dev_err(hba->dev, + "%s: query attribute, opcode %d, idn %d, failed with error %d after %d retires\n", + __func__, opcode, idn, ret, retries); + return ret; +} + +static int __ufshcd_query_descriptor(struct ufs_hba *hba, + enum query_opcode opcode, + enum desc_idn idn, u8 index, u8 selector, + u8 *desc_buf, int *buf_len) +{ + struct ufs_query_req *request = NULL; + struct ufs_query_res *response = NULL; + int err; + + if (!desc_buf) { + dev_err(hba->dev, "%s: descriptor buffer required for opcode 0x%x\n", + __func__, opcode); + err = -EINVAL; + goto out; + } + + if (*buf_len < QUERY_DESC_MIN_SIZE || *buf_len > QUERY_DESC_MAX_SIZE) { + dev_err(hba->dev, "%s: descriptor buffer size (%d) is out of range\n", + __func__, *buf_len); + err = -EINVAL; + goto out; + } + + ufshcd_init_query(hba, &request, &response, opcode, idn, index, + selector); + hba->dev_cmd.query.descriptor = desc_buf; + request->upiu_req.length = cpu_to_be16(*buf_len); + + switch (opcode) { + case UPIU_QUERY_OPCODE_WRITE_DESC: + request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST; + break; + case UPIU_QUERY_OPCODE_READ_DESC: + request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST; + break; + default: + dev_err(hba->dev, "%s: Expected query descriptor opcode but got = 0x%.2x\n", + __func__, opcode); + err = -EINVAL; + goto out; + } + + err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT); + + if (err) { + dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n", + __func__, opcode, idn, index, err); + goto out; + } + + hba->dev_cmd.query.descriptor = NULL; + *buf_len = be16_to_cpu(response->upiu_res.length); + +out: + return err; +} + +/** + * ufshcd_query_descriptor_retry - API function for sending descriptor requests + */ +int ufshcd_query_descriptor_retry(struct ufs_hba *hba, enum query_opcode opcode, + enum desc_idn idn, u8 index, u8 selector, + u8 *desc_buf, int *buf_len) +{ + int err; + int retries; + + for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) { + err = __ufshcd_query_descriptor(hba, opcode, idn, index, + selector, desc_buf, buf_len); + if (!err || err == -EINVAL) + break; + } + + return err; +} + +/** + * ufshcd_read_desc_length - read the specified descriptor length from header + */ +static int ufshcd_read_desc_length(struct ufs_hba *hba, enum desc_idn desc_id, + int desc_index, int *desc_length) +{ + int ret; + u8 header[QUERY_DESC_HDR_SIZE]; + int header_len = QUERY_DESC_HDR_SIZE; + + if (desc_id >= QUERY_DESC_IDN_MAX) + return -EINVAL; + + ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC, + desc_id, desc_index, 0, header, + &header_len); + + if (ret) { + dev_err(hba->dev, "%s: Failed to get descriptor header id %d", + __func__, desc_id); + return ret; + } else if (desc_id != header[QUERY_DESC_DESC_TYPE_OFFSET]) { + dev_warn(hba->dev, "%s: descriptor header id %d and desc_id %d mismatch", + __func__, header[QUERY_DESC_DESC_TYPE_OFFSET], + desc_id); + ret = -EINVAL; + } + + *desc_length = header[QUERY_DESC_LENGTH_OFFSET]; + + return ret; +} + +static void ufshcd_init_desc_sizes(struct ufs_hba *hba) +{ + int err; + + err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_DEVICE, 0, + &hba->desc_size.dev_desc); + if (err) + hba->desc_size.dev_desc = QUERY_DESC_DEVICE_DEF_SIZE; + + err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_POWER, 0, + &hba->desc_size.pwr_desc); + if (err) + hba->desc_size.pwr_desc = QUERY_DESC_POWER_DEF_SIZE; + + err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_INTERCONNECT, 0, + &hba->desc_size.interc_desc); + if (err) + hba->desc_size.interc_desc = QUERY_DESC_INTERCONNECT_DEF_SIZE; + + err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_CONFIGURATION, 0, + &hba->desc_size.conf_desc); + if (err) + hba->desc_size.conf_desc = QUERY_DESC_CONFIGURATION_DEF_SIZE; + + err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_UNIT, 0, + &hba->desc_size.unit_desc); + if (err) + hba->desc_size.unit_desc = QUERY_DESC_UNIT_DEF_SIZE; + + err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_GEOMETRY, 0, + &hba->desc_size.geom_desc); + if (err) + hba->desc_size.geom_desc = QUERY_DESC_GEOMETRY_DEF_SIZE; + + err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_HEALTH, 0, + &hba->desc_size.hlth_desc); + if (err) + hba->desc_size.hlth_desc = QUERY_DESC_HEALTH_DEF_SIZE; +} + +/** + * ufshcd_map_desc_id_to_length - map descriptor IDN to its length + * + */ +int ufshcd_map_desc_id_to_length(struct ufs_hba *hba, enum desc_idn desc_id, + int *desc_len) +{ + switch (desc_id) { + case QUERY_DESC_IDN_DEVICE: + *desc_len = hba->desc_size.dev_desc; + break; + case QUERY_DESC_IDN_POWER: + *desc_len = hba->desc_size.pwr_desc; + break; + case QUERY_DESC_IDN_GEOMETRY: + *desc_len = hba->desc_size.geom_desc; + break; + case QUERY_DESC_IDN_CONFIGURATION: + *desc_len = hba->desc_size.conf_desc; + break; + case QUERY_DESC_IDN_UNIT: + *desc_len = hba->desc_size.unit_desc; + break; + case QUERY_DESC_IDN_INTERCONNECT: + *desc_len = hba->desc_size.interc_desc; + break; + case QUERY_DESC_IDN_STRING: + *desc_len = QUERY_DESC_MAX_SIZE; + break; + case QUERY_DESC_IDN_HEALTH: + *desc_len = hba->desc_size.hlth_desc; + break; + case QUERY_DESC_IDN_RFU_0: + case QUERY_DESC_IDN_RFU_1: + *desc_len = 0; + break; + default: + *desc_len = 0; + return -EINVAL; + } + return 0; +} +EXPORT_SYMBOL(ufshcd_map_desc_id_to_length); + +/** + * ufshcd_read_desc_param - read the specified descriptor parameter + * + */ +int ufshcd_read_desc_param(struct ufs_hba *hba, enum desc_idn desc_id, + int desc_index, u8 param_offset, u8 *param_read_buf, + u8 param_size) +{ + int ret; + u8 *desc_buf; + int buff_len; + bool is_kmalloc = true; + + /* Safety check */ + if (desc_id >= QUERY_DESC_IDN_MAX || !param_size) + return -EINVAL; + + /* Get the max length of descriptor from structure filled up at probe + * time. + */ + ret = ufshcd_map_desc_id_to_length(hba, desc_id, &buff_len); + + /* Sanity checks */ + if (ret || !buff_len) { + dev_err(hba->dev, "%s: Failed to get full descriptor length", + __func__); + return ret; + } + + /* Check whether we need temp memory */ + if (param_offset != 0 || param_size < buff_len) { + desc_buf = kmalloc(buff_len, GFP_KERNEL); + if (!desc_buf) + return -ENOMEM; + } else { + desc_buf = param_read_buf; + is_kmalloc = false; + } + + /* Request for full descriptor */ + ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC, + desc_id, desc_index, 0, desc_buf, + &buff_len); + + if (ret) { + dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d, desc_index %d, param_offset %d, ret %d", + __func__, desc_id, desc_index, param_offset, ret); + goto out; + } + + /* Sanity check */ + if (desc_buf[QUERY_DESC_DESC_TYPE_OFFSET] != desc_id) { + dev_err(hba->dev, "%s: invalid desc_id %d in descriptor header", + __func__, desc_buf[QUERY_DESC_DESC_TYPE_OFFSET]); + ret = -EINVAL; + goto out; + } + + /* Check wherher we will not copy more data, than available */ + if (is_kmalloc && param_size > buff_len) + param_size = buff_len; + + if (is_kmalloc) + memcpy(param_read_buf, &desc_buf[param_offset], param_size); +out: + if (is_kmalloc) + kfree(desc_buf); + return ret; +} + +/* replace non-printable or non-ASCII characters with spaces */ +static inline void ufshcd_remove_non_printable(uint8_t *val) +{ + if (!val) + return; + + if (*val < 0x20 || *val > 0x7e) + *val = ' '; +} + +/** + * ufshcd_uic_pwr_ctrl - executes UIC commands (which affects the link power + * state) and waits for it to take effect. + * + */ +static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd) +{ + unsigned long start = 0; + u8 status; + int ret; + + ret = ufshcd_send_uic_cmd(hba, cmd); + if (ret) { + dev_err(hba->dev, + "pwr ctrl cmd 0x%x with mode 0x%x uic error %d\n", + cmd->command, cmd->argument3, ret); + + return ret; + } + + start = get_timer(0); + do { + status = ufshcd_get_upmcrs(hba); + if (get_timer(start) > UFS_UIC_CMD_TIMEOUT) { + dev_err(hba->dev, + "pwr ctrl cmd 0x%x failed, host upmcrs:0x%x\n", + cmd->command, status); + ret = (status != PWR_OK) ? status : -1; + break; + } + } while (status != PWR_LOCAL); + + return ret; +} + +/** + * ufshcd_uic_change_pwr_mode - Perform the UIC power mode change + * using DME_SET primitives. + */ +static int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode) +{ + struct uic_command uic_cmd = {0}; + int ret; + + uic_cmd.command = UIC_CMD_DME_SET; + uic_cmd.argument1 = UIC_ARG_MIB(PA_PWRMODE); + uic_cmd.argument3 = mode; + ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd); + + return ret; +} + +static +void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufs_hba *hba, + struct scsi_cmd *pccb, u32 upiu_flags) +{ + struct utp_upiu_req *ucd_req_ptr = hba->ucd_req_ptr; + unsigned int cdb_len; + + /* command descriptor fields */ + ucd_req_ptr->header.dword_0 = + UPIU_HEADER_DWORD(UPIU_TRANSACTION_COMMAND, upiu_flags, + pccb->lun, TASK_TAG); + ucd_req_ptr->header.dword_1 = + UPIU_HEADER_DWORD(UPIU_COMMAND_SET_TYPE_SCSI, 0, 0, 0); + + /* Total EHS length and Data segment length will be zero */ + ucd_req_ptr->header.dword_2 = 0; + + ucd_req_ptr->sc.exp_data_transfer_len = cpu_to_be32(pccb->datalen); + + cdb_len = min_t(unsigned short, pccb->cmdlen, UFS_CDB_SIZE); + memset(ucd_req_ptr->sc.cdb, 0, UFS_CDB_SIZE); + memcpy(ucd_req_ptr->sc.cdb, pccb->cmd, cdb_len); + + memset(hba->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp)); +} + +static inline void prepare_prdt_desc(struct ufshcd_sg_entry *entry, + unsigned char *buf, ulong len) +{ + entry->size = cpu_to_le32(len) | GENMASK(1, 0); + entry->base_addr = cpu_to_le32(lower_32_bits((unsigned long)buf)); + entry->upper_addr = cpu_to_le32(upper_32_bits((unsigned long)buf)); +} + +static void prepare_prdt_table(struct ufs_hba *hba, struct scsi_cmd *pccb) +{ + struct utp_transfer_req_desc *req_desc = hba->utrdl; + struct ufshcd_sg_entry *prd_table = hba->ucd_prdt_ptr; + ulong datalen = pccb->datalen; + int table_length; + u8 *buf; + int i; + + if (!datalen) { + req_desc->prd_table_length = 0; + return; + } + + table_length = DIV_ROUND_UP(pccb->datalen, MAX_PRDT_ENTRY); + buf = pccb->pdata; + i = table_length; + while (--i) { + prepare_prdt_desc(&prd_table[table_length - i - 1], buf, + MAX_PRDT_ENTRY - 1); + buf += MAX_PRDT_ENTRY; + datalen -= MAX_PRDT_ENTRY; + } + + prepare_prdt_desc(&prd_table[table_length - i - 1], buf, datalen - 1); + + req_desc->prd_table_length = table_length; +} + +static int ufs_scsi_exec(struct udevice *scsi_dev, struct scsi_cmd *pccb) +{ + struct ufs_hba *hba = dev_get_uclass_priv(scsi_dev->parent); + struct utp_transfer_req_desc *req_desc = hba->utrdl; + u32 upiu_flags; + int ocs, result = 0; + u8 scsi_status; + + ufshcd_prepare_req_desc_hdr(req_desc, &upiu_flags, pccb->dma_dir); + ufshcd_prepare_utp_scsi_cmd_upiu(hba, pccb, upiu_flags); + prepare_prdt_table(hba, pccb); + + ufshcd_send_command(hba, TASK_TAG); + + ocs = ufshcd_get_tr_ocs(hba); + switch (ocs) { + case OCS_SUCCESS: + result = ufshcd_get_req_rsp(hba->ucd_rsp_ptr); + switch (result) { + case UPIU_TRANSACTION_RESPONSE: + result = ufshcd_get_rsp_upiu_result(hba->ucd_rsp_ptr); + + scsi_status = result & MASK_SCSI_STATUS; + if (scsi_status) + return -EINVAL; + + break; + case UPIU_TRANSACTION_REJECT_UPIU: + /* TODO: handle Reject UPIU Response */ + dev_err(hba->dev, + "Reject UPIU not fully implemented\n"); + return -EINVAL; + default: + dev_err(hba->dev, + "Unexpected request response code = %x\n", + result); + return -EINVAL; + } + break; + default: + dev_err(hba->dev, "OCS error from controller = %x\n", ocs); + return -EINVAL; + } + + return 0; +} + +static inline int ufshcd_read_desc(struct ufs_hba *hba, enum desc_idn desc_id, + int desc_index, u8 *buf, u32 size) +{ + return ufshcd_read_desc_param(hba, desc_id, desc_index, 0, buf, size); +} + +static int ufshcd_read_device_desc(struct ufs_hba *hba, u8 *buf, u32 size) +{ + return ufshcd_read_desc(hba, QUERY_DESC_IDN_DEVICE, 0, buf, size); +} + +/** + * ufshcd_read_string_desc - read string descriptor + * + */ +int ufshcd_read_string_desc(struct ufs_hba *hba, int desc_index, + u8 *buf, u32 size, bool ascii) +{ + int err = 0; + + err = ufshcd_read_desc(hba, QUERY_DESC_IDN_STRING, desc_index, buf, + size); + + if (err) { + dev_err(hba->dev, "%s: reading String Desc failed after %d retries. err = %d\n", + __func__, QUERY_REQ_RETRIES, err); + goto out; + } + + if (ascii) { + int desc_len; + int ascii_len; + int i; + u8 *buff_ascii; + + desc_len = buf[0]; + /* remove header and divide by 2 to move from UTF16 to UTF8 */ + ascii_len = (desc_len - QUERY_DESC_HDR_SIZE) / 2 + 1; + if (size < ascii_len + QUERY_DESC_HDR_SIZE) { + dev_err(hba->dev, "%s: buffer allocated size is too small\n", + __func__); + err = -ENOMEM; + goto out; + } + + buff_ascii = kmalloc(ascii_len, GFP_KERNEL); + if (!buff_ascii) { + err = -ENOMEM; + goto out; + } + + /* + * the descriptor contains string in UTF16 format + * we need to convert to utf-8 so it can be displayed + */ + utf16_to_utf8(buff_ascii, + (uint16_t *)&buf[QUERY_DESC_HDR_SIZE], ascii_len); + + /* replace non-printable or non-ASCII characters with spaces */ + for (i = 0; i < ascii_len; i++) + ufshcd_remove_non_printable(&buff_ascii[i]); + + memset(buf + QUERY_DESC_HDR_SIZE, 0, + size - QUERY_DESC_HDR_SIZE); + memcpy(buf + QUERY_DESC_HDR_SIZE, buff_ascii, ascii_len); + buf[QUERY_DESC_LENGTH_OFFSET] = ascii_len + QUERY_DESC_HDR_SIZE; + kfree(buff_ascii); + } +out: + return err; +} + +static int ufs_get_device_desc(struct ufs_hba *hba, + struct ufs_dev_desc *dev_desc) +{ + int err; + size_t buff_len; + u8 model_index; + u8 *desc_buf; + + buff_len = max_t(size_t, hba->desc_size.dev_desc, + QUERY_DESC_MAX_SIZE + 1); + desc_buf = kmalloc(buff_len, GFP_KERNEL); + if (!desc_buf) { + err = -ENOMEM; + goto out; + } + + err = ufshcd_read_device_desc(hba, desc_buf, hba->desc_size.dev_desc); + if (err) { + dev_err(hba->dev, "%s: Failed reading Device Desc. err = %d\n", + __func__, err); + goto out; + } + + /* + * getting vendor (manufacturerID) and Bank Index in big endian + * format + */ + dev_desc->wmanufacturerid = desc_buf[DEVICE_DESC_PARAM_MANF_ID] << 8 | + desc_buf[DEVICE_DESC_PARAM_MANF_ID + 1]; + + model_index = desc_buf[DEVICE_DESC_PARAM_PRDCT_NAME]; + + /* Zero-pad entire buffer for string termination. */ + memset(desc_buf, 0, buff_len); + + err = ufshcd_read_string_desc(hba, model_index, desc_buf, + QUERY_DESC_MAX_SIZE, true/*ASCII*/); + if (err) { + dev_err(hba->dev, "%s: Failed reading Product Name. err = %d\n", + __func__, err); + goto out; + } + + desc_buf[QUERY_DESC_MAX_SIZE] = '\0'; + strlcpy(dev_desc->model, (char *)(desc_buf + QUERY_DESC_HDR_SIZE), + min_t(u8, desc_buf[QUERY_DESC_LENGTH_OFFSET], + MAX_MODEL_LEN)); + + /* Null terminate the model string */ + dev_desc->model[MAX_MODEL_LEN] = '\0'; + +out: + kfree(desc_buf); + return err; +} + +/** + * ufshcd_get_max_pwr_mode - reads the max power mode negotiated with device + */ +static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba) +{ + struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info; + + if (hba->max_pwr_info.is_valid) + return 0; + + pwr_info->pwr_tx = FAST_MODE; + pwr_info->pwr_rx = FAST_MODE; + pwr_info->hs_rate = PA_HS_MODE_B; + + /* Get the connected lane count */ + ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDRXDATALANES), + &pwr_info->lane_rx); + ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES), + &pwr_info->lane_tx); + + if (!pwr_info->lane_rx || !pwr_info->lane_tx) { + dev_err(hba->dev, "%s: invalid connected lanes value. rx=%d, tx=%d\n", + __func__, pwr_info->lane_rx, pwr_info->lane_tx); + return -EINVAL; + } + + /* + * First, get the maximum gears of HS speed. + * If a zero value, it means there is no HSGEAR capability. + * Then, get the maximum gears of PWM speed. + */ + ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), &pwr_info->gear_rx); + if (!pwr_info->gear_rx) { + ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR), + &pwr_info->gear_rx); + if (!pwr_info->gear_rx) { + dev_err(hba->dev, "%s: invalid max pwm rx gear read = %d\n", + __func__, pwr_info->gear_rx); + return -EINVAL; + } + pwr_info->pwr_rx = SLOW_MODE; + } + + ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), + &pwr_info->gear_tx); + if (!pwr_info->gear_tx) { + ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR), + &pwr_info->gear_tx); + if (!pwr_info->gear_tx) { + dev_err(hba->dev, "%s: invalid max pwm tx gear read = %d\n", + __func__, pwr_info->gear_tx); + return -EINVAL; + } + pwr_info->pwr_tx = SLOW_MODE; + } + + hba->max_pwr_info.is_valid = true; + return 0; +} + +static int ufshcd_change_power_mode(struct ufs_hba *hba, + struct ufs_pa_layer_attr *pwr_mode) +{ + int ret; + + /* if already configured to the requested pwr_mode */ + if (pwr_mode->gear_rx == hba->pwr_info.gear_rx && + pwr_mode->gear_tx == hba->pwr_info.gear_tx && + pwr_mode->lane_rx == hba->pwr_info.lane_rx && + pwr_mode->lane_tx == hba->pwr_info.lane_tx && + pwr_mode->pwr_rx == hba->pwr_info.pwr_rx && + pwr_mode->pwr_tx == hba->pwr_info.pwr_tx && + pwr_mode->hs_rate == hba->pwr_info.hs_rate) { + dev_dbg(hba->dev, "%s: power already configured\n", __func__); + return 0; + } + + /* + * Configure attributes for power mode change with below. + * - PA_RXGEAR, PA_ACTIVERXDATALANES, PA_RXTERMINATION, + * - PA_TXGEAR, PA_ACTIVETXDATALANES, PA_TXTERMINATION, + * - PA_HSSERIES + */ + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXGEAR), pwr_mode->gear_rx); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVERXDATALANES), + pwr_mode->lane_rx); + if (pwr_mode->pwr_rx == FASTAUTO_MODE || pwr_mode->pwr_rx == FAST_MODE) + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), TRUE); + else + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), FALSE); + + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXGEAR), pwr_mode->gear_tx); + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVETXDATALANES), + pwr_mode->lane_tx); + if (pwr_mode->pwr_tx == FASTAUTO_MODE || pwr_mode->pwr_tx == FAST_MODE) + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), TRUE); + else + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), FALSE); + + if (pwr_mode->pwr_rx == FASTAUTO_MODE || + pwr_mode->pwr_tx == FASTAUTO_MODE || + pwr_mode->pwr_rx == FAST_MODE || + pwr_mode->pwr_tx == FAST_MODE) + ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HSSERIES), + pwr_mode->hs_rate); + + ret = ufshcd_uic_change_pwr_mode(hba, pwr_mode->pwr_rx << 4 | + pwr_mode->pwr_tx); + + if (ret) { + dev_err(hba->dev, + "%s: power mode change failed %d\n", __func__, ret); + + return ret; + } + + /* Copy new Power Mode to power info */ + memcpy(&hba->pwr_info, pwr_mode, sizeof(struct ufs_pa_layer_attr)); + + return ret; +} + +/** + * ufshcd_verify_dev_init() - Verify device initialization + * + */ +static int ufshcd_verify_dev_init(struct ufs_hba *hba) +{ + int retries; + int err; + + for (retries = NOP_OUT_RETRIES; retries > 0; retries--) { + err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_NOP, + NOP_OUT_TIMEOUT); + if (!err || err == -ETIMEDOUT) + break; + + dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err); + } + + if (err) + dev_err(hba->dev, "%s: NOP OUT failed %d\n", __func__, err); + + return err; +} + +/** + * ufshcd_complete_dev_init() - checks device readiness + */ +static int ufshcd_complete_dev_init(struct ufs_hba *hba) +{ + int i; + int err; + bool flag_res = 1; + + err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG, + QUERY_FLAG_IDN_FDEVICEINIT, NULL); + if (err) { + dev_err(hba->dev, + "%s setting fDeviceInit flag failed with error %d\n", + __func__, err); + goto out; + } + + /* poll for max. 1000 iterations for fDeviceInit flag to clear */ + for (i = 0; i < 1000 && !err && flag_res; i++) + err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG, + QUERY_FLAG_IDN_FDEVICEINIT, + &flag_res); + + if (err) + dev_err(hba->dev, + "%s reading fDeviceInit flag failed with error %d\n", + __func__, err); + else if (flag_res) + dev_err(hba->dev, + "%s fDeviceInit was not cleared by the device\n", + __func__); + +out: + return err; +} + +static void ufshcd_def_desc_sizes(struct ufs_hba *hba) +{ + hba->desc_size.dev_desc = QUERY_DESC_DEVICE_DEF_SIZE; + hba->desc_size.pwr_desc = QUERY_DESC_POWER_DEF_SIZE; + hba->desc_size.interc_desc = QUERY_DESC_INTERCONNECT_DEF_SIZE; + hba->desc_size.conf_desc = QUERY_DESC_CONFIGURATION_DEF_SIZE; + hba->desc_size.unit_desc = QUERY_DESC_UNIT_DEF_SIZE; + hba->desc_size.geom_desc = QUERY_DESC_GEOMETRY_DEF_SIZE; + hba->desc_size.hlth_desc = QUERY_DESC_HEALTH_DEF_SIZE; +} + +int ufs_start(struct ufs_hba *hba) +{ + struct ufs_dev_desc card = {0}; + int ret; + + ret = ufshcd_link_startup(hba); + if (ret) + return ret; + + ret = ufshcd_verify_dev_init(hba); + if (ret) + return ret; + + ret = ufshcd_complete_dev_init(hba); + if (ret) + return ret; + + /* Init check for device descriptor sizes */ + ufshcd_init_desc_sizes(hba); + + ret = ufs_get_device_desc(hba, &card); + if (ret) { + dev_err(hba->dev, "%s: Failed getting device info. err = %d\n", + __func__, ret); + + return ret; + } + + if (ufshcd_get_max_pwr_mode(hba)) { + dev_err(hba->dev, + "%s: Failed getting max supported power mode\n", + __func__); + } else { + ret = ufshcd_change_power_mode(hba, &hba->max_pwr_info.info); + if (ret) { + dev_err(hba->dev, "%s: Failed setting power mode, err = %d\n", + __func__, ret); + + return ret; + } + + printf("Device at %s up at:", hba->dev->name); + ufshcd_print_pwr_info(hba); + } + + return 0; +} + +int ufshcd_probe(struct udevice *ufs_dev, struct ufs_hba_ops *hba_ops) +{ + struct ufs_hba *hba = dev_get_uclass_priv(ufs_dev); + struct scsi_platdata *scsi_plat; + struct udevice *scsi_dev; + int err; + + device_find_first_child(ufs_dev, &scsi_dev); + if (!scsi_dev) + return -ENODEV; + + scsi_plat = dev_get_uclass_platdata(scsi_dev); + scsi_plat->max_id = UFSHCD_MAX_ID; + scsi_plat->max_lun = UFS_MAX_LUNS; + scsi_plat->max_bytes_per_req = UFS_MAX_BYTES; + + hba->dev = ufs_dev; + hba->ops = hba_ops; + hba->mmio_base = (void *)dev_read_addr(ufs_dev); + + /* Set descriptor lengths to specification defaults */ + ufshcd_def_desc_sizes(hba); + + ufshcd_ops_init(hba); + + /* Read capabilties registers */ + hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES); + + /* Get UFS version supported by the controller */ + hba->version = ufshcd_get_ufs_version(hba); + if (hba->version != UFSHCI_VERSION_10 && + hba->version != UFSHCI_VERSION_11 && + hba->version != UFSHCI_VERSION_20 && + hba->version != UFSHCI_VERSION_21) + dev_err(hba->dev, "invalid UFS version 0x%x\n", + hba->version); + + /* Get Interrupt bit mask per version */ + hba->intr_mask = ufshcd_get_intr_mask(hba); + + /* Allocate memory for host memory space */ + err = ufshcd_memory_alloc(hba); + if (err) { + dev_err(hba->dev, "Memory allocation failed\n"); + return err; + } + + /* Configure Local data structures */ + ufshcd_host_memory_configure(hba); + + /* + * In order to avoid any spurious interrupt immediately after + * registering UFS controller interrupt handler, clear any pending UFS + * interrupt status and disable all the UFS interrupts. + */ + ufshcd_writel(hba, ufshcd_readl(hba, REG_INTERRUPT_STATUS), + REG_INTERRUPT_STATUS); + ufshcd_writel(hba, 0, REG_INTERRUPT_ENABLE); + + err = ufshcd_hba_enable(hba); + if (err) { + dev_err(hba->dev, "Host controller enable failed\n"); + return err; + } + + err = ufs_start(hba); + if (err) + return err; + + return 0; +} + +int ufs_scsi_bind(struct udevice *ufs_dev, struct udevice **scsi_devp) +{ + int ret = device_bind_driver(ufs_dev, "ufs_scsi", "ufs_scsi", + scsi_devp); + + return ret; +} + +static struct scsi_ops ufs_ops = { + .exec = ufs_scsi_exec, +}; + +int ufs_probe_dev(int index) +{ + struct udevice *dev; + + return uclass_get_device(UCLASS_UFS, index, &dev); +} + +int ufs_probe(void) +{ + struct udevice *dev; + int ret, i; + + for (i = 0;; i++) { + ret = uclass_get_device(UCLASS_UFS, i, &dev); + if (ret == -ENODEV) + break; + } + + return 0; +} + +U_BOOT_DRIVER(ufs_scsi) = { + .id = UCLASS_SCSI, + .name = "ufs_scsi", + .ops = &ufs_ops, +}; diff --git a/drivers/usb/cdns3/gadget.c b/drivers/usb/cdns3/gadget.c new file mode 100644 index 0000000000..8377eb458b --- /dev/null +++ b/drivers/usb/cdns3/gadget.c @@ -0,0 +1,2763 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Cadence USBSS DRD Driver - gadget side. + * + * Copyright (C) 2018-2019 Cadence Design Systems. + * Copyright (C) 2017-2018 NXP + * + * Authors: Pawel Jez , + * Pawel Laszczak + * Peter Chen + */ + +/* + * Work around 1: + * At some situations, the controller may get stale data address in TRB + * at below sequences: + * 1. Controller read TRB includes data address + * 2. Software updates TRBs includes data address and Cycle bit + * 3. Controller read TRB which includes Cycle bit + * 4. DMA run with stale data address + * + * To fix this problem, driver needs to make the first TRB in TD as invalid. + * After preparing all TRBs driver needs to check the position of DMA and + * if the DMA point to the first just added TRB and doorbell is 1, + * then driver must defer making this TRB as valid. This TRB will be make + * as valid during adding next TRB only if DMA is stopped or at TRBERR + * interrupt. + * + * Issue has been fixed in DEV_VER_V3 version of controller. + * + * Work around 2: + * Controller for OUT endpoints has shared on-chip buffers for all incoming + * packets, including ep0out. It's FIFO buffer, so packets must be handle by DMA + * in correct order. If the first packet in the buffer will not be handled, + * then the following packets directed for other endpoints and functions + * will be blocked. + * Additionally the packets directed to one endpoint can block entire on-chip + * buffers. In this case transfer to other endpoints also will blocked. + * + * To resolve this issue after raising the descriptor missing interrupt + * driver prepares internal usb_request object and use it to arm DMA transfer. + * + * The problematic situation was observed in case when endpoint has been enabled + * but no usb_request were queued. Driver try detects such endpoints and will + * use this workaround only for these endpoint. + * + * Driver use limited number of buffer. This number can be set by macro + * CDNS3_WA2_NUM_BUFFERS. + * + * Such blocking situation was observed on ACM gadget. For this function + * host send OUT data packet but ACM function is not prepared for this packet. + * It's cause that buffer placed in on chip memory block transfer to other + * endpoints. + * + * Issue has been fixed in DEV_VER_V2 version of controller. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "core.h" +#include "gadget-export.h" +#include "gadget.h" +#include "trace.h" +#include "drd.h" + +#define readl_poll_timeout_atomic readl_poll_timeout +#define usleep_range(a, b) udelay((b)) + +static int __cdns3_gadget_ep_queue(struct usb_ep *ep, + struct usb_request *request, + gfp_t gfp_flags); + +/** + * cdns3_set_register_bit - set bit in given register. + * @ptr: address of device controller register to be read and changed + * @mask: bits requested to set + */ +void cdns3_set_register_bit(void __iomem *ptr, u32 mask) +{ + mask = readl(ptr) | mask; + writel(mask, ptr); +} + +/** + * cdns3_ep_addr_to_index - Macro converts endpoint address to + * index of endpoint object in cdns3_device.eps[] container + * @ep_addr: endpoint address for which endpoint object is required + * + */ +u8 cdns3_ep_addr_to_index(u8 ep_addr) +{ + return (((ep_addr & 0x7F)) + ((ep_addr & USB_DIR_IN) ? 16 : 0)); +} + +static int cdns3_get_dma_pos(struct cdns3_device *priv_dev, + struct cdns3_endpoint *priv_ep) +{ + int dma_index; + + dma_index = readl(&priv_dev->regs->ep_traddr) - priv_ep->trb_pool_dma; + + return dma_index / TRB_SIZE; +} + +/** + * cdns3_next_request - returns next request from list + * @list: list containing requests + * + * Returns request or NULL if no requests in list + */ +struct usb_request *cdns3_next_request(struct list_head *list) +{ + return list_first_entry_or_null(list, struct usb_request, list); +} + +/** + * cdns3_next_align_buf - returns next buffer from list + * @list: list containing buffers + * + * Returns buffer or NULL if no buffers in list + */ +struct cdns3_aligned_buf *cdns3_next_align_buf(struct list_head *list) +{ + return list_first_entry_or_null(list, struct cdns3_aligned_buf, list); +} + +/** + * cdns3_next_priv_request - returns next request from list + * @list: list containing requests + * + * Returns request or NULL if no requests in list + */ +struct cdns3_request *cdns3_next_priv_request(struct list_head *list) +{ + return list_first_entry_or_null(list, struct cdns3_request, list); +} + +/** + * select_ep - selects endpoint + * @priv_dev: extended gadget object + * @ep: endpoint address + */ +void cdns3_select_ep(struct cdns3_device *priv_dev, u32 ep) +{ + if (priv_dev->selected_ep == ep) + return; + + priv_dev->selected_ep = ep; + writel(ep, &priv_dev->regs->ep_sel); +} + +dma_addr_t cdns3_trb_virt_to_dma(struct cdns3_endpoint *priv_ep, + struct cdns3_trb *trb) +{ + u32 offset = (char *)trb - (char *)priv_ep->trb_pool; + + return priv_ep->trb_pool_dma + offset; +} + +int cdns3_ring_size(struct cdns3_endpoint *priv_ep) +{ + switch (priv_ep->type) { + case USB_ENDPOINT_XFER_ISOC: + return TRB_ISO_RING_SIZE; + case USB_ENDPOINT_XFER_CONTROL: + return TRB_CTRL_RING_SIZE; + default: + return TRB_RING_SIZE; + } +} + +/** + * cdns3_allocate_trb_pool - Allocates TRB's pool for selected endpoint + * @priv_ep: endpoint object + * + * Function will return 0 on success or -ENOMEM on allocation error + */ +int cdns3_allocate_trb_pool(struct cdns3_endpoint *priv_ep) +{ + int ring_size = cdns3_ring_size(priv_ep); + struct cdns3_trb *link_trb; + + if (!priv_ep->trb_pool) { + priv_ep->trb_pool = + dma_alloc_coherent(ring_size, + (unsigned long *)&priv_ep->trb_pool_dma); + if (!priv_ep->trb_pool) + return -ENOMEM; + } else { + memset(priv_ep->trb_pool, 0, ring_size); + } + + if (!priv_ep->num) + return 0; + + priv_ep->num_trbs = ring_size / TRB_SIZE; + /* Initialize the last TRB as Link TRB. */ + link_trb = (priv_ep->trb_pool + (priv_ep->num_trbs - 1)); + link_trb->buffer = TRB_BUFFER(priv_ep->trb_pool_dma); + link_trb->control = TRB_CYCLE | TRB_TYPE(TRB_LINK) | TRB_TOGGLE; + + return 0; +} + +static void cdns3_free_trb_pool(struct cdns3_endpoint *priv_ep) +{ + if (priv_ep->trb_pool) { + dma_free_coherent(priv_ep->trb_pool); + priv_ep->trb_pool = NULL; + } +} + +/** + * cdns3_ep_stall_flush - Stalls and flushes selected endpoint + * @priv_ep: endpoint object + * + * Endpoint must be selected before call to this function + */ +static void cdns3_ep_stall_flush(struct cdns3_endpoint *priv_ep) +{ + struct cdns3_device *priv_dev = priv_ep->cdns3_dev; + int val; + + trace_cdns3_halt(priv_ep, 1, 1); + + writel(EP_CMD_DFLUSH | EP_CMD_ERDY | EP_CMD_SSTALL, + &priv_dev->regs->ep_cmd); + + /* wait for DFLUSH cleared */ + readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val, + !(val & EP_CMD_DFLUSH), 1000); + priv_ep->flags |= EP_STALLED; + priv_ep->flags &= ~EP_STALL_PENDING; +} + +/** + * cdns3_hw_reset_eps_config - reset endpoints configuration kept by controller. + * @priv_dev: extended gadget object + */ +void cdns3_hw_reset_eps_config(struct cdns3_device *priv_dev) +{ + writel(USB_CONF_CFGRST, &priv_dev->regs->usb_conf); + + cdns3_allow_enable_l1(priv_dev, 0); + priv_dev->hw_configured_flag = 0; + priv_dev->onchip_used_size = 0; + priv_dev->out_mem_is_allocated = 0; + priv_dev->wait_for_setup = 0; +} + +/** + * cdns3_ep_inc_trb - increment a trb index. + * @index: Pointer to the TRB index to increment. + * @cs: Cycle state + * @trb_in_seg: number of TRBs in segment + * + * The index should never point to the link TRB. After incrementing, + * if it is point to the link TRB, wrap around to the beginning and revert + * cycle state bit The + * link TRB is always at the last TRB entry. + */ +static void cdns3_ep_inc_trb(int *index, u8 *cs, int trb_in_seg) +{ + (*index)++; + if (*index == (trb_in_seg - 1)) { + *index = 0; + *cs ^= 1; + } +} + +/** + * cdns3_ep_inc_enq - increment endpoint's enqueue pointer + * @priv_ep: The endpoint whose enqueue pointer we're incrementing + */ +static void cdns3_ep_inc_enq(struct cdns3_endpoint *priv_ep) +{ + priv_ep->free_trbs--; + cdns3_ep_inc_trb(&priv_ep->enqueue, &priv_ep->pcs, priv_ep->num_trbs); +} + +/** + * cdns3_ep_inc_deq - increment endpoint's dequeue pointer + * @priv_ep: The endpoint whose dequeue pointer we're incrementing + */ +static void cdns3_ep_inc_deq(struct cdns3_endpoint *priv_ep) +{ + priv_ep->free_trbs++; + cdns3_ep_inc_trb(&priv_ep->dequeue, &priv_ep->ccs, priv_ep->num_trbs); +} + +void cdns3_move_deq_to_next_trb(struct cdns3_request *priv_req) +{ + struct cdns3_endpoint *priv_ep = priv_req->priv_ep; + int current_trb = priv_req->start_trb; + + while (current_trb != priv_req->end_trb) { + cdns3_ep_inc_deq(priv_ep); + current_trb = priv_ep->dequeue; + } + + cdns3_ep_inc_deq(priv_ep); +} + +/** + * cdns3_allow_enable_l1 - enable/disable permits to transition to L1. + * @priv_dev: Extended gadget object + * @enable: Enable/disable permit to transition to L1. + * + * If bit USB_CONF_L1EN is set and device receive Extended Token packet, + * then controller answer with ACK handshake. + * If bit USB_CONF_L1DS is set and device receive Extended Token packet, + * then controller answer with NYET handshake. + */ +void cdns3_allow_enable_l1(struct cdns3_device *priv_dev, int enable) +{ + if (enable) + writel(USB_CONF_L1EN, &priv_dev->regs->usb_conf); + else + writel(USB_CONF_L1DS, &priv_dev->regs->usb_conf); +} + +enum usb_device_speed cdns3_get_speed(struct cdns3_device *priv_dev) +{ + u32 reg; + + reg = readl(&priv_dev->regs->usb_sts); + + if (DEV_SUPERSPEED(reg)) + return USB_SPEED_SUPER; + else if (DEV_HIGHSPEED(reg)) + return USB_SPEED_HIGH; + else if (DEV_FULLSPEED(reg)) + return USB_SPEED_FULL; + else if (DEV_LOWSPEED(reg)) + return USB_SPEED_LOW; + return USB_SPEED_UNKNOWN; +} + +/** + * cdns3_start_all_request - add to ring all request not started + * @priv_dev: Extended gadget object + * @priv_ep: The endpoint for whom request will be started. + * + * Returns return ENOMEM if transfer ring i not enough TRBs to start + * all requests. + */ +static int cdns3_start_all_request(struct cdns3_device *priv_dev, + struct cdns3_endpoint *priv_ep) +{ + struct usb_request *request; + int ret = 0; + + while (!list_empty(&priv_ep->deferred_req_list)) { + request = cdns3_next_request(&priv_ep->deferred_req_list); + + ret = cdns3_ep_run_transfer(priv_ep, request); + if (ret) + return ret; + + list_del(&request->list); + list_add_tail(&request->list, + &priv_ep->pending_req_list); + } + + priv_ep->flags &= ~EP_RING_FULL; + return ret; +} + +/* + * WA2: Set flag for all not ISOC OUT endpoints. If this flag is set + * driver try to detect whether endpoint need additional internal + * buffer for unblocking on-chip FIFO buffer. This flag will be cleared + * if before first DESCMISS interrupt the DMA will be armed. + */ +#define cdns3_wa2_enable_detection(priv_dev, ep_priv, reg) do { \ + if (!priv_ep->dir && priv_ep->type != USB_ENDPOINT_XFER_ISOC) { \ + priv_ep->flags |= EP_QUIRK_EXTRA_BUF_DET; \ + (reg) |= EP_STS_EN_DESCMISEN; \ + } } while (0) + +/** + * cdns3_wa2_descmiss_copy_data copy data from internal requests to + * request queued by class driver. + * @priv_ep: extended endpoint object + * @request: request object + */ +static void cdns3_wa2_descmiss_copy_data(struct cdns3_endpoint *priv_ep, + struct usb_request *request) +{ + struct usb_request *descmiss_req; + struct cdns3_request *descmiss_priv_req; + + while (!list_empty(&priv_ep->wa2_descmiss_req_list)) { + int chunk_end; + int length; + + descmiss_priv_req = + cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list); + descmiss_req = &descmiss_priv_req->request; + + /* driver can't touch pending request */ + if (descmiss_priv_req->flags & REQUEST_PENDING) + break; + + chunk_end = descmiss_priv_req->flags & REQUEST_INTERNAL_CH; + length = request->actual + descmiss_req->actual; + + request->status = descmiss_req->status; + + if (length <= request->length) { + memcpy(&((u8 *)request->buf)[request->actual], + descmiss_req->buf, + descmiss_req->actual); + request->actual = length; + } else { + /* It should never occur */ + request->status = -ENOMEM; + } + + list_del_init(&descmiss_priv_req->list); + + kfree(descmiss_req->buf); + cdns3_gadget_ep_free_request(&priv_ep->endpoint, descmiss_req); + --priv_ep->wa2_counter; + + if (!chunk_end) + break; + } +} + +struct usb_request *cdns3_wa2_gadget_giveback(struct cdns3_device *priv_dev, + struct cdns3_endpoint *priv_ep, + struct cdns3_request *priv_req) +{ + if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN && + priv_req->flags & REQUEST_INTERNAL) { + struct usb_request *req; + + req = cdns3_next_request(&priv_ep->deferred_req_list); + + priv_ep->descmis_req = NULL; + + if (!req) + return NULL; + + cdns3_wa2_descmiss_copy_data(priv_ep, req); + if (!(priv_ep->flags & EP_QUIRK_END_TRANSFER) && + req->length != req->actual) { + /* wait for next part of transfer */ + return NULL; + } + + if (req->status == -EINPROGRESS) + req->status = 0; + + list_del_init(&req->list); + cdns3_start_all_request(priv_dev, priv_ep); + return req; + } + + return &priv_req->request; +} + +int cdns3_wa2_gadget_ep_queue(struct cdns3_device *priv_dev, + struct cdns3_endpoint *priv_ep, + struct cdns3_request *priv_req) +{ + int deferred = 0; + + /* + * If transfer was queued before DESCMISS appear than we + * can disable handling of DESCMISS interrupt. Driver assumes that it + * can disable special treatment for this endpoint. + */ + if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) { + u32 reg; + + cdns3_select_ep(priv_dev, priv_ep->num | priv_ep->dir); + priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET; + reg = readl(&priv_dev->regs->ep_sts_en); + reg &= ~EP_STS_EN_DESCMISEN; + trace_cdns3_wa2(priv_ep, "workaround disabled\n"); + writel(reg, &priv_dev->regs->ep_sts_en); + } + + if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) { + u8 pending_empty = list_empty(&priv_ep->pending_req_list); + u8 descmiss_empty = list_empty(&priv_ep->wa2_descmiss_req_list); + + /* + * DESCMISS transfer has been finished, so data will be + * directly copied from internal allocated usb_request + * objects. + */ + if (pending_empty && !descmiss_empty && + !(priv_req->flags & REQUEST_INTERNAL)) { + cdns3_wa2_descmiss_copy_data(priv_ep, + &priv_req->request); + + trace_cdns3_wa2(priv_ep, "get internal stored data"); + + list_add_tail(&priv_req->request.list, + &priv_ep->pending_req_list); + cdns3_gadget_giveback(priv_ep, priv_req, + priv_req->request.status); + + /* + * Intentionally driver returns positive value as + * correct value. It informs that transfer has + * been finished. + */ + return EINPROGRESS; + } + + /* + * Driver will wait for completion DESCMISS transfer, + * before starts new, not DESCMISS transfer. + */ + if (!pending_empty && !descmiss_empty) { + trace_cdns3_wa2(priv_ep, "wait for pending transfer\n"); + deferred = 1; + } + + if (priv_req->flags & REQUEST_INTERNAL) + list_add_tail(&priv_req->list, + &priv_ep->wa2_descmiss_req_list); + } + + return deferred; +} + +static void cdns3_wa2_remove_old_request(struct cdns3_endpoint *priv_ep) +{ + struct cdns3_request *priv_req; + + while (!list_empty(&priv_ep->wa2_descmiss_req_list)) { + u8 chain; + + priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list); + chain = !!(priv_req->flags & REQUEST_INTERNAL_CH); + + trace_cdns3_wa2(priv_ep, "removes eldest request"); + + kfree(priv_req->request.buf); + cdns3_gadget_ep_free_request(&priv_ep->endpoint, + &priv_req->request); + list_del_init(&priv_req->list); + --priv_ep->wa2_counter; + + if (!chain) + break; + } +} + +/** + * cdns3_wa2_descmissing_packet - handles descriptor missing event. + * @priv_dev: extended gadget object + * + * This function is used only for WA2. For more information see Work around 2 + * description. + */ +static void cdns3_wa2_descmissing_packet(struct cdns3_endpoint *priv_ep) +{ + struct cdns3_request *priv_req; + struct usb_request *request; + + if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) { + priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET; + priv_ep->flags |= EP_QUIRK_EXTRA_BUF_EN; + } + + trace_cdns3_wa2(priv_ep, "Description Missing detected\n"); + + if (priv_ep->wa2_counter >= CDNS3_WA2_NUM_BUFFERS) + cdns3_wa2_remove_old_request(priv_ep); + + request = cdns3_gadget_ep_alloc_request(&priv_ep->endpoint, + GFP_ATOMIC); + if (!request) + goto err; + + priv_req = to_cdns3_request(request); + priv_req->flags |= REQUEST_INTERNAL; + + /* if this field is still assigned it indicate that transfer related + * with this request has not been finished yet. Driver in this + * case simply allocate next request and assign flag REQUEST_INTERNAL_CH + * flag to previous one. It will indicate that current request is + * part of the previous one. + */ + if (priv_ep->descmis_req) + priv_ep->descmis_req->flags |= REQUEST_INTERNAL_CH; + + priv_req->request.buf = kzalloc(CDNS3_DESCMIS_BUF_SIZE, + GFP_ATOMIC); + priv_ep->wa2_counter++; + + if (!priv_req->request.buf) { + cdns3_gadget_ep_free_request(&priv_ep->endpoint, request); + goto err; + } + + priv_req->request.length = CDNS3_DESCMIS_BUF_SIZE; + priv_ep->descmis_req = priv_req; + + __cdns3_gadget_ep_queue(&priv_ep->endpoint, + &priv_ep->descmis_req->request, + GFP_ATOMIC); + + return; + +err: + dev_err(priv_ep->cdns3_dev->dev, + "Failed: No sufficient memory for DESCMIS\n"); +} + +/** + * cdns3_gadget_giveback - call struct usb_request's ->complete callback + * @priv_ep: The endpoint to whom the request belongs to + * @priv_req: The request we're giving back + * @status: completion code for the request + * + * Must be called with controller's lock held and interrupts disabled. This + * function will unmap @req and call its ->complete() callback to notify upper + * layers that it has completed. + */ +void cdns3_gadget_giveback(struct cdns3_endpoint *priv_ep, + struct cdns3_request *priv_req, + int status) +{ + struct cdns3_device *priv_dev = priv_ep->cdns3_dev; + struct usb_request *request = &priv_req->request; + + list_del_init(&request->list); + + if (request->status == -EINPROGRESS) + request->status = status; + + usb_gadget_unmap_request(&priv_dev->gadget, request, + priv_ep->dir); + + if ((priv_req->flags & REQUEST_UNALIGNED) && + priv_ep->dir == USB_DIR_OUT && !request->status) + memcpy(request->buf, priv_req->aligned_buf->buf, + request->length); + + priv_req->flags &= ~(REQUEST_PENDING | REQUEST_UNALIGNED); + trace_cdns3_gadget_giveback(priv_req); + + if (priv_dev->dev_ver < DEV_VER_V2) { + request = cdns3_wa2_gadget_giveback(priv_dev, priv_ep, + priv_req); + if (!request) + return; + } + + if (request->complete) { + spin_unlock(&priv_dev->lock); + usb_gadget_giveback_request(&priv_ep->endpoint, + request); + spin_lock(&priv_dev->lock); + } + + if (request->buf == priv_dev->zlp_buf) + cdns3_gadget_ep_free_request(&priv_ep->endpoint, request); +} + +void cdns3_wa1_restore_cycle_bit(struct cdns3_endpoint *priv_ep) +{ + /* Work around for stale data address in TRB*/ + if (priv_ep->wa1_set) { + trace_cdns3_wa1(priv_ep, "restore cycle bit"); + + priv_ep->wa1_set = 0; + priv_ep->wa1_trb_index = 0xFFFF; + if (priv_ep->wa1_cycle_bit) { + priv_ep->wa1_trb->control = + priv_ep->wa1_trb->control | 0x1; + } else { + priv_ep->wa1_trb->control = + priv_ep->wa1_trb->control & ~0x1; + } + } +} + +static void cdns3_free_aligned_request_buf(struct cdns3_device *priv_dev) +{ + struct cdns3_aligned_buf *buf, *tmp; + unsigned long flags; + + spin_lock_irqsave(&priv_dev->lock, flags); + + list_for_each_entry_safe(buf, tmp, &priv_dev->aligned_buf_list, list) { + if (!buf->in_use) { + list_del(&buf->list); + + /* + * Re-enable interrupts to free DMA capable memory. + * Driver can't free this memory with disabled + * interrupts. + */ + spin_unlock_irqrestore(&priv_dev->lock, flags); + dma_free_coherent(buf->buf); + kfree(buf); + spin_lock_irqsave(&priv_dev->lock, flags); + } + } + + spin_unlock_irqrestore(&priv_dev->lock, flags); +} + +static int cdns3_prepare_aligned_request_buf(struct cdns3_request *priv_req) +{ + struct cdns3_endpoint *priv_ep = priv_req->priv_ep; + struct cdns3_device *priv_dev = priv_ep->cdns3_dev; + struct cdns3_aligned_buf *buf; + + /* check if buffer is aligned to 8. */ + if (!((uintptr_t)priv_req->request.buf & 0x7)) + return 0; + + buf = priv_req->aligned_buf; + + if (!buf || priv_req->request.length > buf->size) { + buf = kzalloc(sizeof(*buf), GFP_ATOMIC); + if (!buf) + return -ENOMEM; + + buf->size = priv_req->request.length; + + buf->buf = dma_alloc_coherent(buf->size, + (unsigned long *)&buf->dma); + if (!buf->buf) { + kfree(buf); + return -ENOMEM; + } + + if (priv_req->aligned_buf) { + trace_cdns3_free_aligned_request(priv_req); + priv_req->aligned_buf->in_use = 0; +#ifndef __UBOOT__ + queue_work(system_freezable_wq, + &priv_dev->aligned_buf_wq); +#else + cdns3_free_aligned_request_buf(priv_dev); +#endif + } + + buf->in_use = 1; + priv_req->aligned_buf = buf; + + list_add_tail(&buf->list, + &priv_dev->aligned_buf_list); + } + + if (priv_ep->dir == USB_DIR_IN) { + memcpy(buf->buf, priv_req->request.buf, + priv_req->request.length); + } + + priv_req->flags |= REQUEST_UNALIGNED; + trace_cdns3_prepare_aligned_request(priv_req); + + return 0; +} + +static int cdns3_wa1_update_guard(struct cdns3_endpoint *priv_ep, + struct cdns3_trb *trb) +{ + struct cdns3_device *priv_dev = priv_ep->cdns3_dev; + + if (!priv_ep->wa1_set) { + u32 doorbell; + + doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY); + + if (doorbell) { + priv_ep->wa1_cycle_bit = priv_ep->pcs ? TRB_CYCLE : 0; + priv_ep->wa1_set = 1; + priv_ep->wa1_trb = trb; + priv_ep->wa1_trb_index = priv_ep->enqueue; + trace_cdns3_wa1(priv_ep, "set guard"); + return 0; + } + } + return 1; +} + +static void cdns3_wa1_tray_restore_cycle_bit(struct cdns3_device *priv_dev, + struct cdns3_endpoint *priv_ep) +{ + int dma_index; + u32 doorbell; + + doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY); + dma_index = cdns3_get_dma_pos(priv_dev, priv_ep); + + if (!doorbell || dma_index != priv_ep->wa1_trb_index) + cdns3_wa1_restore_cycle_bit(priv_ep); +} + +/** + * cdns3_ep_run_transfer - start transfer on no-default endpoint hardware + * @priv_ep: endpoint object + * + * Returns zero on success or negative value on failure + */ +int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep, + struct usb_request *request) +{ + struct cdns3_device *priv_dev = priv_ep->cdns3_dev; + struct cdns3_request *priv_req; + struct cdns3_trb *trb; + dma_addr_t trb_dma; + u32 togle_pcs = 1; + int sg_iter = 0; + int num_trb = 1; + int address; + u32 control; + int pcs; + + if (num_trb > priv_ep->free_trbs) { + priv_ep->flags |= EP_RING_FULL; + return -ENOBUFS; + } + + priv_req = to_cdns3_request(request); + address = priv_ep->endpoint.desc->bEndpointAddress; + + priv_ep->flags |= EP_PENDING_REQUEST; + + /* must allocate buffer aligned to 8 */ + if (priv_req->flags & REQUEST_UNALIGNED) + trb_dma = priv_req->aligned_buf->dma; + else + trb_dma = request->dma; + + trb = priv_ep->trb_pool + priv_ep->enqueue; + priv_req->start_trb = priv_ep->enqueue; + priv_req->trb = trb; + + cdns3_select_ep(priv_ep->cdns3_dev, address); + + /* prepare ring */ + if ((priv_ep->enqueue + num_trb) >= (priv_ep->num_trbs - 1)) { + struct cdns3_trb *link_trb; + int doorbell, dma_index; + u32 ch_bit = 0; + + doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY); + dma_index = cdns3_get_dma_pos(priv_dev, priv_ep); + + /* Driver can't update LINK TRB if it is current processed. */ + if (doorbell && dma_index == priv_ep->num_trbs - 1) { + priv_ep->flags |= EP_DEFERRED_DRDY; + return -ENOBUFS; + } + + /*updating C bt in Link TRB before starting DMA*/ + link_trb = priv_ep->trb_pool + (priv_ep->num_trbs - 1); + /* + * For TRs size equal 2 enabling TRB_CHAIN for epXin causes + * that DMA stuck at the LINK TRB. + * On the other hand, removing TRB_CHAIN for longer TRs for + * epXout cause that DMA stuck after handling LINK TRB. + * To eliminate this strange behavioral driver set TRB_CHAIN + * bit only for TR size > 2. + */ + if (priv_ep->type == USB_ENDPOINT_XFER_ISOC || + TRBS_PER_SEGMENT > 2) + ch_bit = TRB_CHAIN; + + link_trb->control = ((priv_ep->pcs) ? TRB_CYCLE : 0) | + TRB_TYPE(TRB_LINK) | TRB_TOGGLE | ch_bit; + } + + if (priv_dev->dev_ver <= DEV_VER_V2) + togle_pcs = cdns3_wa1_update_guard(priv_ep, trb); + + /* set incorrect Cycle Bit for first trb*/ + control = priv_ep->pcs ? 0 : TRB_CYCLE; + + do { + u32 length; + u16 td_size = 0; + + /* fill TRB */ + control |= TRB_TYPE(TRB_NORMAL); + trb->buffer = TRB_BUFFER(trb_dma); + + length = request->length; + + if (likely(priv_dev->dev_ver >= DEV_VER_V2)) + td_size = DIV_ROUND_UP(length, + priv_ep->endpoint.maxpacket); + + trb->length = TRB_BURST_LEN(priv_ep->trb_burst_size) | + TRB_LEN(length); + if (priv_dev->gadget.speed == USB_SPEED_SUPER) + trb->length |= TRB_TDL_SS_SIZE(td_size); + else + control |= TRB_TDL_HS_SIZE(td_size); + + pcs = priv_ep->pcs ? TRB_CYCLE : 0; + + /* + * first trb should be prepared as last to avoid processing + * transfer to early + */ + if (sg_iter != 0) + control |= pcs; + + if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir) { + control |= TRB_IOC | TRB_ISP; + } else { + /* for last element in TD or in SG list */ + if (sg_iter == (num_trb - 1) && sg_iter != 0) + control |= pcs | TRB_IOC | TRB_ISP; + } + + if (sg_iter) + trb->control = control; + else + priv_req->trb->control = control; + + control = 0; + ++sg_iter; + priv_req->end_trb = priv_ep->enqueue; + cdns3_ep_inc_enq(priv_ep); + trb = priv_ep->trb_pool + priv_ep->enqueue; + } while (sg_iter < num_trb); + + trb = priv_req->trb; + + priv_req->flags |= REQUEST_PENDING; + + if (sg_iter == 1) + trb->control |= TRB_IOC | TRB_ISP; + + /* + * Memory barrier - cycle bit must be set before other filds in trb. + */ + dmb(); + + /* give the TD to the consumer*/ + if (togle_pcs) + trb->control = trb->control ^ 1; + + if (priv_dev->dev_ver <= DEV_VER_V2) + cdns3_wa1_tray_restore_cycle_bit(priv_dev, priv_ep); + + trace_cdns3_prepare_trb(priv_ep, priv_req->trb); + + /* + * Memory barrier - Cycle Bit must be set before trb->length and + * trb->buffer fields. + */ + dmb(); + + /* + * For DMULT mode we can set address to transfer ring only once after + * enabling endpoint. + */ + if (priv_ep->flags & EP_UPDATE_EP_TRBADDR) { + /* + * Until SW is not ready to handle the OUT transfer the ISO OUT + * Endpoint should be disabled (EP_CFG.ENABLE = 0). + * EP_CFG_ENABLE must be set before updating ep_traddr. + */ + if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir && + !(priv_ep->flags & EP_QUIRK_ISO_OUT_EN)) { + priv_ep->flags |= EP_QUIRK_ISO_OUT_EN; + cdns3_set_register_bit(&priv_dev->regs->ep_cfg, + EP_CFG_ENABLE); + } + + writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma + + priv_req->start_trb * TRB_SIZE), + &priv_dev->regs->ep_traddr); + + priv_ep->flags &= ~EP_UPDATE_EP_TRBADDR; + } + + if (!priv_ep->wa1_set && !(priv_ep->flags & EP_STALLED)) { + trace_cdns3_ring(priv_ep); + /*clearing TRBERR and EP_STS_DESCMIS before seting DRDY*/ + writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts); + writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd); + trace_cdns3_doorbell_epx(priv_ep->name, + readl(&priv_dev->regs->ep_traddr)); + } + + /* WORKAROUND for transition to L0 */ + __cdns3_gadget_wakeup(priv_dev); + + return 0; +} + +void cdns3_set_hw_configuration(struct cdns3_device *priv_dev) +{ + struct cdns3_endpoint *priv_ep; + struct usb_ep *ep; + int val; + + if (priv_dev->hw_configured_flag) + return; + + writel(USB_CONF_CFGSET, &priv_dev->regs->usb_conf); + writel(EP_CMD_ERDY | EP_CMD_REQ_CMPL, &priv_dev->regs->ep_cmd); + + cdns3_set_register_bit(&priv_dev->regs->usb_conf, + USB_CONF_U1EN | USB_CONF_U2EN); + + /* wait until configuration set */ + readl_poll_timeout_atomic(&priv_dev->regs->usb_sts, val, + val & USB_STS_CFGSTS_MASK, 100); + + priv_dev->hw_configured_flag = 1; + + list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) { + priv_ep = ep_to_cdns3_ep(ep); + if (priv_ep->flags & EP_ENABLED) + cdns3_start_all_request(priv_dev, priv_ep); + } +} + +/** + * cdns3_request_handled - check whether request has been handled by DMA + * + * @priv_ep: extended endpoint object. + * @priv_req: request object for checking + * + * Endpoint must be selected before invoking this function. + * + * Returns false if request has not been handled by DMA, else returns true. + * + * SR - start ring + * ER - end ring + * DQ = priv_ep->dequeue - dequeue position + * EQ = priv_ep->enqueue - enqueue position + * ST = priv_req->start_trb - index of first TRB in transfer ring + * ET = priv_req->end_trb - index of last TRB in transfer ring + * CI = current_index - index of processed TRB by DMA. + * + * As first step, function checks if cycle bit for priv_req->start_trb is + * correct. + * + * some rules: + * 1. priv_ep->dequeue never exceed current_index. + * 2 priv_ep->enqueue never exceed priv_ep->dequeue + * 3. exception: priv_ep->enqueue == priv_ep->dequeue + * and priv_ep->free_trbs is zero. + * This case indicate that TR is full. + * + * Then We can split recognition into two parts: + * Case 1 - priv_ep->dequeue < current_index + * SR ... EQ ... DQ ... CI ... ER + * SR ... DQ ... CI ... EQ ... ER + * + * Request has been handled by DMA if ST and ET is between DQ and CI. + * + * Case 2 - priv_ep->dequeue > current_index + * This situation take place when CI go through the LINK TRB at the end of + * transfer ring. + * SR ... CI ... EQ ... DQ ... ER + * + * Request has been handled by DMA if ET is less then CI or + * ET is greater or equal DQ. + */ +static bool cdns3_request_handled(struct cdns3_endpoint *priv_ep, + struct cdns3_request *priv_req) +{ + struct cdns3_device *priv_dev = priv_ep->cdns3_dev; + struct cdns3_trb *trb = priv_req->trb; + int current_index = 0; + int handled = 0; + int doorbell; + + current_index = cdns3_get_dma_pos(priv_dev, priv_ep); + doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY); + + trb = &priv_ep->trb_pool[priv_req->start_trb]; + + if ((trb->control & TRB_CYCLE) != priv_ep->ccs) + goto finish; + + if (doorbell == 1 && current_index == priv_ep->dequeue) + goto finish; + + /* The corner case for TRBS_PER_SEGMENT equal 2). */ + if (TRBS_PER_SEGMENT == 2 && priv_ep->type != USB_ENDPOINT_XFER_ISOC) { + handled = 1; + goto finish; + } + + if (priv_ep->enqueue == priv_ep->dequeue && + priv_ep->free_trbs == 0) { + handled = 1; + } else if (priv_ep->dequeue < current_index) { + if ((current_index == (priv_ep->num_trbs - 1)) && + !priv_ep->dequeue) + goto finish; + + if (priv_req->end_trb >= priv_ep->dequeue && + priv_req->end_trb < current_index) + handled = 1; + } else if (priv_ep->dequeue > current_index) { + if (priv_req->end_trb < current_index || + priv_req->end_trb >= priv_ep->dequeue) + handled = 1; + } + +finish: + trace_cdns3_request_handled(priv_req, current_index, handled); + + return handled; +} + +static void cdns3_transfer_completed(struct cdns3_device *priv_dev, + struct cdns3_endpoint *priv_ep) +{ + struct cdns3_request *priv_req; + struct usb_request *request; + struct cdns3_trb *trb; + + while (!list_empty(&priv_ep->pending_req_list)) { + request = cdns3_next_request(&priv_ep->pending_req_list); + priv_req = to_cdns3_request(request); + + /* Re-select endpoint. It could be changed by other CPU during + * handling usb_gadget_giveback_request. + */ +#ifndef __UBOOT__ + cdns3_select_ep(priv_dev, priv_ep->endpoint.address); +#else + cdns3_select_ep(priv_dev, + priv_ep->endpoint.desc->bEndpointAddress); +#endif + + if (!cdns3_request_handled(priv_ep, priv_req)) + goto prepare_next_td; + + trb = priv_ep->trb_pool + priv_ep->dequeue; + trace_cdns3_complete_trb(priv_ep, trb); + + if (trb != priv_req->trb) + dev_warn(priv_dev->dev, + "request_trb=0x%p, queue_trb=0x%p\n", + priv_req->trb, trb); + + request->actual = TRB_LEN(le32_to_cpu(trb->length)); + cdns3_move_deq_to_next_trb(priv_req); + cdns3_gadget_giveback(priv_ep, priv_req, 0); + + if (priv_ep->type != USB_ENDPOINT_XFER_ISOC && + TRBS_PER_SEGMENT == 2) + break; + } + priv_ep->flags &= ~EP_PENDING_REQUEST; + +prepare_next_td: + if (!(priv_ep->flags & EP_STALLED) && + !(priv_ep->flags & EP_STALL_PENDING)) + cdns3_start_all_request(priv_dev, priv_ep); +} + +void cdns3_rearm_transfer(struct cdns3_endpoint *priv_ep, u8 rearm) +{ + struct cdns3_device *priv_dev = priv_ep->cdns3_dev; + + cdns3_wa1_restore_cycle_bit(priv_ep); + + if (rearm) { + trace_cdns3_ring(priv_ep); + + /* Cycle Bit must be updated before arming DMA. */ + dmb(); + writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd); + + __cdns3_gadget_wakeup(priv_dev); + + trace_cdns3_doorbell_epx(priv_ep->name, + readl(&priv_dev->regs->ep_traddr)); + } +} + +/** + * cdns3_check_ep_interrupt_proceed - Processes interrupt related to endpoint + * @priv_ep: endpoint object + * + * Returns 0 + */ +static int cdns3_check_ep_interrupt_proceed(struct cdns3_endpoint *priv_ep) +{ + struct cdns3_device *priv_dev = priv_ep->cdns3_dev; + u32 ep_sts_reg; + +#ifndef __UBOOT__ + cdns3_select_ep(priv_dev, priv_ep->endpoint.address); +#else + cdns3_select_ep(priv_dev, priv_ep->endpoint.desc->bEndpointAddress); +#endif + + trace_cdns3_epx_irq(priv_dev, priv_ep); + + ep_sts_reg = readl(&priv_dev->regs->ep_sts); + writel(ep_sts_reg, &priv_dev->regs->ep_sts); + + if (ep_sts_reg & EP_STS_TRBERR) { + if (priv_ep->flags & EP_STALL_PENDING && + !(ep_sts_reg & EP_STS_DESCMIS && + priv_dev->dev_ver < DEV_VER_V2)) { + cdns3_ep_stall_flush(priv_ep); + } + + /* + * For isochronous transfer driver completes request on + * IOC or on TRBERR. IOC appears only when device receive + * OUT data packet. If host disable stream or lost some packet + * then the only way to finish all queued transfer is to do it + * on TRBERR event. + */ + if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && + !priv_ep->wa1_set) { + if (!priv_ep->dir) { + u32 ep_cfg = readl(&priv_dev->regs->ep_cfg); + + ep_cfg &= ~EP_CFG_ENABLE; + writel(ep_cfg, &priv_dev->regs->ep_cfg); + priv_ep->flags &= ~EP_QUIRK_ISO_OUT_EN; + } + cdns3_transfer_completed(priv_dev, priv_ep); + } else if (!(priv_ep->flags & EP_STALLED) && + !(priv_ep->flags & EP_STALL_PENDING)) { + if (priv_ep->flags & EP_DEFERRED_DRDY) { + priv_ep->flags &= ~EP_DEFERRED_DRDY; + cdns3_start_all_request(priv_dev, priv_ep); + } else { + cdns3_rearm_transfer(priv_ep, + priv_ep->wa1_set); + } + } + } + + if ((ep_sts_reg & EP_STS_IOC) || (ep_sts_reg & EP_STS_ISP)) { + if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) { + if (ep_sts_reg & EP_STS_ISP) + priv_ep->flags |= EP_QUIRK_END_TRANSFER; + else + priv_ep->flags &= ~EP_QUIRK_END_TRANSFER; + } + + cdns3_transfer_completed(priv_dev, priv_ep); + } + + /* + * WA2: this condition should only be meet when + * priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET or + * priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN. + * In other cases this interrupt will be disabled/ + */ + if (ep_sts_reg & EP_STS_DESCMIS && priv_dev->dev_ver < DEV_VER_V2 && + !(priv_ep->flags & EP_STALLED)) + cdns3_wa2_descmissing_packet(priv_ep); + + return 0; +} + +static void cdns3_disconnect_gadget(struct cdns3_device *priv_dev) +{ + if (priv_dev->gadget_driver && priv_dev->gadget_driver->disconnect) { + spin_unlock(&priv_dev->lock); + priv_dev->gadget_driver->disconnect(&priv_dev->gadget); + spin_lock(&priv_dev->lock); + } +} + +/** + * cdns3_check_usb_interrupt_proceed - Processes interrupt related to device + * @priv_dev: extended gadget object + * @usb_ists: bitmap representation of device's reported interrupts + * (usb_ists register value) + */ +static void cdns3_check_usb_interrupt_proceed(struct cdns3_device *priv_dev, + u32 usb_ists) +{ + int speed = 0; + + trace_cdns3_usb_irq(priv_dev, usb_ists); + if (usb_ists & USB_ISTS_L1ENTI) { + /* + * WORKAROUND: CDNS3 controller has issue with hardware resuming + * from L1. To fix it, if any DMA transfer is pending driver + * must starts driving resume signal immediately. + */ + if (readl(&priv_dev->regs->drbl)) + __cdns3_gadget_wakeup(priv_dev); + } + + /* Connection detected */ + if (usb_ists & (USB_ISTS_CON2I | USB_ISTS_CONI)) { + speed = cdns3_get_speed(priv_dev); + priv_dev->gadget.speed = speed; + usb_gadget_set_state(&priv_dev->gadget, USB_STATE_POWERED); + cdns3_ep0_config(priv_dev); + } + + /* Disconnection detected */ + if (usb_ists & (USB_ISTS_DIS2I | USB_ISTS_DISI)) { + cdns3_disconnect_gadget(priv_dev); + priv_dev->gadget.speed = USB_SPEED_UNKNOWN; + usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED); + cdns3_hw_reset_eps_config(priv_dev); + } + + if (usb_ists & (USB_ISTS_L2ENTI | USB_ISTS_U3ENTI)) { + if (priv_dev->gadget_driver && + priv_dev->gadget_driver->suspend) { + spin_unlock(&priv_dev->lock); + priv_dev->gadget_driver->suspend(&priv_dev->gadget); + spin_lock(&priv_dev->lock); + } + } + + if (usb_ists & (USB_ISTS_L2EXTI | USB_ISTS_U3EXTI)) { + if (priv_dev->gadget_driver && + priv_dev->gadget_driver->resume) { + spin_unlock(&priv_dev->lock); + priv_dev->gadget_driver->resume(&priv_dev->gadget); + spin_lock(&priv_dev->lock); + } + } + + /* reset*/ + if (usb_ists & (USB_ISTS_UWRESI | USB_ISTS_UHRESI | USB_ISTS_U2RESI)) { + if (priv_dev->gadget_driver) { + spin_unlock(&priv_dev->lock); + usb_gadget_udc_reset(&priv_dev->gadget, + priv_dev->gadget_driver); + spin_lock(&priv_dev->lock); + + /*read again to check the actual speed*/ + speed = cdns3_get_speed(priv_dev); + priv_dev->gadget.speed = speed; + cdns3_hw_reset_eps_config(priv_dev); + cdns3_ep0_config(priv_dev); + } + } +} + +/** + * cdns3_device_irq_handler- interrupt handler for device part of controller + * + * @irq: irq number for cdns3 core device + * @data: structure of cdns3 + * + * Returns IRQ_HANDLED or IRQ_NONE + */ +static irqreturn_t cdns3_device_irq_handler(int irq, void *data) +{ + struct cdns3_device *priv_dev; + struct cdns3 *cdns = data; + irqreturn_t ret = IRQ_NONE; + u32 reg; + + priv_dev = cdns->gadget_dev; + + /* check USB device interrupt */ + reg = readl(&priv_dev->regs->usb_ists); + if (reg) { + /* After masking interrupts the new interrupts won't be + * reported in usb_ists/ep_ists. In order to not lose some + * of them driver disables only detected interrupts. + * They will be enabled ASAP after clearing source of + * interrupt. This an unusual behavior only applies to + * usb_ists register. + */ + reg = ~reg & readl(&priv_dev->regs->usb_ien); + /* mask deferred interrupt. */ + writel(reg, &priv_dev->regs->usb_ien); + ret = IRQ_WAKE_THREAD; + } + + /* check endpoint interrupt */ + reg = readl(&priv_dev->regs->ep_ists); + if (reg) { + writel(0, &priv_dev->regs->ep_ien); + ret = IRQ_WAKE_THREAD; + } + + return ret; +} + +/** + * cdns3_device_thread_irq_handler- interrupt handler for device part + * of controller + * + * @irq: irq number for cdns3 core device + * @data: structure of cdns3 + * + * Returns IRQ_HANDLED or IRQ_NONE + */ +static irqreturn_t cdns3_device_thread_irq_handler(int irq, void *data) +{ + struct cdns3_device *priv_dev; + struct cdns3 *cdns = data; + irqreturn_t ret = IRQ_NONE; + unsigned long flags; + int bit; + u32 reg; + + priv_dev = cdns->gadget_dev; + spin_lock_irqsave(&priv_dev->lock, flags); + + reg = readl(&priv_dev->regs->usb_ists); + if (reg) { + writel(reg, &priv_dev->regs->usb_ists); + writel(USB_IEN_INIT, &priv_dev->regs->usb_ien); + cdns3_check_usb_interrupt_proceed(priv_dev, reg); + ret = IRQ_HANDLED; + } + + reg = readl(&priv_dev->regs->ep_ists); + + /* handle default endpoint OUT */ + if (reg & EP_ISTS_EP_OUT0) { + cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_OUT); + ret = IRQ_HANDLED; + } + + /* handle default endpoint IN */ + if (reg & EP_ISTS_EP_IN0) { + cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_IN); + ret = IRQ_HANDLED; + } + + /* check if interrupt from non default endpoint, if no exit */ + reg &= ~(EP_ISTS_EP_OUT0 | EP_ISTS_EP_IN0); + if (!reg) + goto irqend; + + for_each_set_bit(bit, (unsigned long *)®, + sizeof(u32) * BITS_PER_BYTE) { + cdns3_check_ep_interrupt_proceed(priv_dev->eps[bit]); + ret = IRQ_HANDLED; + } + +irqend: + writel(~0, &priv_dev->regs->ep_ien); + spin_unlock_irqrestore(&priv_dev->lock, flags); + + return ret; +} + +/** + * cdns3_ep_onchip_buffer_reserve - Try to reserve onchip buf for EP + * + * The real reservation will occur during write to EP_CFG register, + * this function is used to check if the 'size' reservation is allowed. + * + * @priv_dev: extended gadget object + * @size: the size (KB) for EP would like to allocate + * @is_in: endpoint direction + * + * Return 0 if the required size can met or negative value on failure + */ +static int cdns3_ep_onchip_buffer_reserve(struct cdns3_device *priv_dev, + int size, int is_in) +{ + int remained; + + /* 2KB are reserved for EP0*/ + remained = priv_dev->onchip_buffers - priv_dev->onchip_used_size - 2; + + if (is_in) { + if (remained < size) + return -EPERM; + + priv_dev->onchip_used_size += size; + } else { + int required; + + /** + * ALL OUT EPs are shared the same chunk onchip memory, so + * driver checks if it already has assigned enough buffers + */ + if (priv_dev->out_mem_is_allocated >= size) + return 0; + + required = size - priv_dev->out_mem_is_allocated; + + if (required > remained) + return -EPERM; + + priv_dev->out_mem_is_allocated += required; + priv_dev->onchip_used_size += required; + } + + return 0; +} + +void cdns3_configure_dmult(struct cdns3_device *priv_dev, + struct cdns3_endpoint *priv_ep) +{ + struct cdns3_usb_regs __iomem *regs = priv_dev->regs; + + /* For dev_ver > DEV_VER_V2 DMULT is configured per endpoint */ + if (priv_dev->dev_ver <= DEV_VER_V2) + writel(USB_CONF_DMULT, ®s->usb_conf); + + if (priv_dev->dev_ver == DEV_VER_V2) + writel(USB_CONF2_EN_TDL_TRB, ®s->usb_conf2); + + if (priv_dev->dev_ver >= DEV_VER_V3 && priv_ep) { + u32 mask; + + if (priv_ep->dir) + mask = BIT(priv_ep->num + 16); + else + mask = BIT(priv_ep->num); + + if (priv_ep->type != USB_ENDPOINT_XFER_ISOC) { + cdns3_set_register_bit(®s->tdl_from_trb, mask); + cdns3_set_register_bit(®s->tdl_beh, mask); + cdns3_set_register_bit(®s->tdl_beh2, mask); + cdns3_set_register_bit(®s->dma_adv_td, mask); + } + + if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir) + cdns3_set_register_bit(®s->tdl_from_trb, mask); + + cdns3_set_register_bit(®s->dtrans, mask); + } +} + +/** + * cdns3_ep_config Configure hardware endpoint + * @priv_ep: extended endpoint object + */ +void cdns3_ep_config(struct cdns3_endpoint *priv_ep) +{ + bool is_iso_ep = (priv_ep->type == USB_ENDPOINT_XFER_ISOC); + struct cdns3_device *priv_dev = priv_ep->cdns3_dev; + u32 bEndpointAddress = priv_ep->num | priv_ep->dir; + u32 max_packet_size = 0; + u8 maxburst = 0; + u32 ep_cfg = 0; + u8 buffering; + u8 mult = 0; + int ret; + + buffering = CDNS3_EP_BUF_SIZE - 1; + + cdns3_configure_dmult(priv_dev, priv_ep); + + switch (priv_ep->type) { + case USB_ENDPOINT_XFER_INT: + ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_INT); + + if ((priv_dev->dev_ver == DEV_VER_V2 && !priv_ep->dir) || + priv_dev->dev_ver > DEV_VER_V2) + ep_cfg |= EP_CFG_TDL_CHK; + break; + case USB_ENDPOINT_XFER_BULK: + ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_BULK); + + if ((priv_dev->dev_ver == DEV_VER_V2 && !priv_ep->dir) || + priv_dev->dev_ver > DEV_VER_V2) + ep_cfg |= EP_CFG_TDL_CHK; + break; + default: + ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_ISOC); + mult = CDNS3_EP_ISO_HS_MULT - 1; + buffering = mult + 1; + } + + switch (priv_dev->gadget.speed) { + case USB_SPEED_FULL: + max_packet_size = is_iso_ep ? 1023 : 64; + break; + case USB_SPEED_HIGH: + max_packet_size = is_iso_ep ? 1024 : 512; + break; + case USB_SPEED_SUPER: + /* It's limitation that driver assumes in driver. */ + mult = 0; + max_packet_size = 1024; + if (priv_ep->type == USB_ENDPOINT_XFER_ISOC) { + maxburst = CDNS3_EP_ISO_SS_BURST - 1; + buffering = (mult + 1) * + (maxburst + 1); + + if (priv_ep->interval > 1) + buffering++; + } else { + maxburst = CDNS3_EP_BUF_SIZE - 1; + } + break; + default: + /* all other speed are not supported */ + return; + } + + if (max_packet_size == 1024) + priv_ep->trb_burst_size = 128; + else if (max_packet_size >= 512) + priv_ep->trb_burst_size = 64; + else + priv_ep->trb_burst_size = 16; + + ret = cdns3_ep_onchip_buffer_reserve(priv_dev, buffering + 1, + !!priv_ep->dir); + if (ret) { + dev_err(priv_dev->dev, "onchip mem is full, ep is invalid\n"); + return; + } + + ep_cfg |= EP_CFG_MAXPKTSIZE(max_packet_size) | + EP_CFG_MULT(mult) | + EP_CFG_BUFFERING(buffering) | + EP_CFG_MAXBURST(maxburst); + + cdns3_select_ep(priv_dev, bEndpointAddress); + writel(ep_cfg, &priv_dev->regs->ep_cfg); + + dev_dbg(priv_dev->dev, "Configure %s: with val %08x\n", + priv_ep->name, ep_cfg); +} + +/* Find correct direction for HW endpoint according to description */ +static int cdns3_ep_dir_is_correct(struct usb_endpoint_descriptor *desc, + struct cdns3_endpoint *priv_ep) +{ + return (priv_ep->endpoint.caps.dir_in && usb_endpoint_dir_in(desc)) || + (priv_ep->endpoint.caps.dir_out && usb_endpoint_dir_out(desc)); +} + +static struct +cdns3_endpoint *cdns3_find_available_ep(struct cdns3_device *priv_dev, + struct usb_endpoint_descriptor *desc) +{ + struct usb_ep *ep; + struct cdns3_endpoint *priv_ep; + + list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) { + unsigned long num; + /* ep name pattern likes epXin or epXout */ + char c[2] = {ep->name[2], '\0'}; + + num = simple_strtoul(c, NULL, 10); + + priv_ep = ep_to_cdns3_ep(ep); + if (cdns3_ep_dir_is_correct(desc, priv_ep)) { + if (!(priv_ep->flags & EP_CLAIMED)) { + priv_ep->num = num; + return priv_ep; + } + } + } + + return ERR_PTR(-ENOENT); +} + +/* + * Cadence IP has one limitation that all endpoints must be configured + * (Type & MaxPacketSize) before setting configuration through hardware + * register, it means we can't change endpoints configuration after + * set_configuration. + * + * This function set EP_CLAIMED flag which is added when the gadget driver + * uses usb_ep_autoconfig to configure specific endpoint; + * When the udc driver receives set_configurion request, + * it goes through all claimed endpoints, and configure all endpoints + * accordingly. + * + * At usb_ep_ops.enable/disable, we only enable and disable endpoint through + * ep_cfg register which can be changed after set_configuration, and do + * some software operation accordingly. + */ +static struct +usb_ep *cdns3_gadget_match_ep(struct usb_gadget *gadget, + struct usb_endpoint_descriptor *desc, + struct usb_ss_ep_comp_descriptor *comp_desc) +{ + struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget); + struct cdns3_endpoint *priv_ep; + unsigned long flags; + + priv_ep = cdns3_find_available_ep(priv_dev, desc); + if (IS_ERR(priv_ep)) { + dev_err(priv_dev->dev, "no available ep\n"); + return NULL; + } + + dev_dbg(priv_dev->dev, "match endpoint: %s\n", priv_ep->name); + + spin_lock_irqsave(&priv_dev->lock, flags); + priv_ep->endpoint.desc = desc; + priv_ep->dir = usb_endpoint_dir_in(desc) ? USB_DIR_IN : USB_DIR_OUT; + priv_ep->type = usb_endpoint_type(desc); + priv_ep->flags |= EP_CLAIMED; + priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0; + + spin_unlock_irqrestore(&priv_dev->lock, flags); + return &priv_ep->endpoint; +} + +/** + * cdns3_gadget_ep_alloc_request Allocates request + * @ep: endpoint object associated with request + * @gfp_flags: gfp flags + * + * Returns allocated request address, NULL on allocation error + */ +struct usb_request *cdns3_gadget_ep_alloc_request(struct usb_ep *ep, + gfp_t gfp_flags) +{ + struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep); + struct cdns3_request *priv_req; + + priv_req = kzalloc(sizeof(*priv_req), gfp_flags); + if (!priv_req) + return NULL; + + priv_req->priv_ep = priv_ep; + + trace_cdns3_alloc_request(priv_req); + return &priv_req->request; +} + +/** + * cdns3_gadget_ep_free_request Free memory occupied by request + * @ep: endpoint object associated with request + * @request: request to free memory + */ +void cdns3_gadget_ep_free_request(struct usb_ep *ep, + struct usb_request *request) +{ + struct cdns3_request *priv_req = to_cdns3_request(request); + + if (priv_req->aligned_buf) + priv_req->aligned_buf->in_use = 0; + + trace_cdns3_free_request(priv_req); + kfree(priv_req); +} + +/** + * cdns3_gadget_ep_enable Enable endpoint + * @ep: endpoint object + * @desc: endpoint descriptor + * + * Returns 0 on success, error code elsewhere + */ +static int cdns3_gadget_ep_enable(struct usb_ep *ep, + const struct usb_endpoint_descriptor *desc) +{ + struct cdns3_endpoint *priv_ep; + struct cdns3_device *priv_dev; + u32 reg = EP_STS_EN_TRBERREN; + u32 bEndpointAddress; + unsigned long flags; + int enable = 1; + int ret; + int val; + + priv_ep = ep_to_cdns3_ep(ep); + priv_dev = priv_ep->cdns3_dev; + + if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) { + dev_dbg(priv_dev->dev, "usbss: invalid parameters\n"); + return -EINVAL; + } + + if (!desc->wMaxPacketSize) { + dev_err(priv_dev->dev, "usbss: missing wMaxPacketSize\n"); + return -EINVAL; + } + + if (WARN_ON(priv_ep->flags & EP_ENABLED)) + return 0; + + spin_lock_irqsave(&priv_dev->lock, flags); + + priv_ep->endpoint.desc = desc; + priv_ep->type = usb_endpoint_type(desc); + priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0; + + if (priv_ep->interval > ISO_MAX_INTERVAL && + priv_ep->type == USB_ENDPOINT_XFER_ISOC) { + dev_err(priv_dev->dev, "Driver is limited to %d period\n", + ISO_MAX_INTERVAL); + + ret = -EINVAL; + goto exit; + } + + ret = cdns3_allocate_trb_pool(priv_ep); + + if (ret) + goto exit; + + bEndpointAddress = priv_ep->num | priv_ep->dir; + cdns3_select_ep(priv_dev, bEndpointAddress); + + trace_cdns3_gadget_ep_enable(priv_ep); + + writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd); + + ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val, + !(val & (EP_CMD_CSTALL | EP_CMD_EPRST)), + 1000); + + if (unlikely(ret)) { + cdns3_free_trb_pool(priv_ep); + ret = -EINVAL; + goto exit; + } + + /* enable interrupt for selected endpoint */ + cdns3_set_register_bit(&priv_dev->regs->ep_ien, + BIT(cdns3_ep_addr_to_index(bEndpointAddress))); + + if (priv_dev->dev_ver < DEV_VER_V2) + cdns3_wa2_enable_detection(priv_dev, priv_ep, reg); + + writel(reg, &priv_dev->regs->ep_sts_en); + + /* + * For some versions of controller at some point during ISO OUT traffic + * DMA reads Transfer Ring for the EP which has never got doorbell. + * This issue was detected only on simulation, but to avoid this issue + * driver add protection against it. To fix it driver enable ISO OUT + * endpoint before setting DRBL. This special treatment of ISO OUT + * endpoints are recommended by controller specification. + */ + if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir) + enable = 0; + + if (enable) + cdns3_set_register_bit(&priv_dev->regs->ep_cfg, EP_CFG_ENABLE); + + ep->desc = desc; + priv_ep->flags &= ~(EP_PENDING_REQUEST | EP_STALLED | EP_STALL_PENDING | + EP_QUIRK_ISO_OUT_EN | EP_QUIRK_EXTRA_BUF_EN); + priv_ep->flags |= EP_ENABLED | EP_UPDATE_EP_TRBADDR; + priv_ep->wa1_set = 0; + priv_ep->enqueue = 0; + priv_ep->dequeue = 0; + reg = readl(&priv_dev->regs->ep_sts); + priv_ep->pcs = !!EP_STS_CCS(reg); + priv_ep->ccs = !!EP_STS_CCS(reg); + /* one TRB is reserved for link TRB used in DMULT mode*/ + priv_ep->free_trbs = priv_ep->num_trbs - 1; +exit: + spin_unlock_irqrestore(&priv_dev->lock, flags); + + return ret; +} + +/** + * cdns3_gadget_ep_disable Disable endpoint + * @ep: endpoint object + * + * Returns 0 on success, error code elsewhere + */ +static int cdns3_gadget_ep_disable(struct usb_ep *ep) +{ + struct cdns3_endpoint *priv_ep; + struct cdns3_request *priv_req; + struct cdns3_device *priv_dev; + struct usb_request *request; + unsigned long flags; + int ret = 0; + u32 ep_cfg; + int val; + + if (!ep) { + pr_err("usbss: invalid parameters\n"); + return -EINVAL; + } + + priv_ep = ep_to_cdns3_ep(ep); + priv_dev = priv_ep->cdns3_dev; + + if (WARN_ON(!(priv_ep->flags & EP_ENABLED))) + return 0; + + spin_lock_irqsave(&priv_dev->lock, flags); + + trace_cdns3_gadget_ep_disable(priv_ep); + + cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress); + + ep_cfg = readl(&priv_dev->regs->ep_cfg); + ep_cfg &= ~EP_CFG_ENABLE; + writel(ep_cfg, &priv_dev->regs->ep_cfg); + + /** + * Driver needs some time before resetting endpoint. + * It need waits for clearing DBUSY bit or for timeout expired. + * 10us is enough time for controller to stop transfer. + */ + readl_poll_timeout_atomic(&priv_dev->regs->ep_sts, val, + !(val & EP_STS_DBUSY), 10); + writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd); + + readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val, + !(val & (EP_CMD_CSTALL | EP_CMD_EPRST)), + 1000); + if (unlikely(ret)) + dev_err(priv_dev->dev, "Timeout: %s resetting failed.\n", + priv_ep->name); + + while (!list_empty(&priv_ep->pending_req_list)) { + request = cdns3_next_request(&priv_ep->pending_req_list); + + cdns3_gadget_giveback(priv_ep, to_cdns3_request(request), + -ESHUTDOWN); + } + + while (!list_empty(&priv_ep->wa2_descmiss_req_list)) { + priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list); + + kfree(priv_req->request.buf); + cdns3_gadget_ep_free_request(&priv_ep->endpoint, + &priv_req->request); + list_del_init(&priv_req->list); + --priv_ep->wa2_counter; + } + + while (!list_empty(&priv_ep->deferred_req_list)) { + request = cdns3_next_request(&priv_ep->deferred_req_list); + + cdns3_gadget_giveback(priv_ep, to_cdns3_request(request), + -ESHUTDOWN); + } + + priv_ep->descmis_req = NULL; + + ep->desc = NULL; + priv_ep->flags &= ~EP_ENABLED; + + spin_unlock_irqrestore(&priv_dev->lock, flags); + + return ret; +} + +/** + * cdns3_gadget_ep_queue Transfer data on endpoint + * @ep: endpoint object + * @request: request object + * @gfp_flags: gfp flags + * + * Returns 0 on success, error code elsewhere + */ +static int __cdns3_gadget_ep_queue(struct usb_ep *ep, + struct usb_request *request, + gfp_t gfp_flags) +{ + struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep); + struct cdns3_device *priv_dev = priv_ep->cdns3_dev; + struct cdns3_request *priv_req; + int ret = 0; + + request->actual = 0; + request->status = -EINPROGRESS; + priv_req = to_cdns3_request(request); + trace_cdns3_ep_queue(priv_req); + + if (priv_dev->dev_ver < DEV_VER_V2) { + ret = cdns3_wa2_gadget_ep_queue(priv_dev, priv_ep, + priv_req); + + if (ret == EINPROGRESS) + return 0; + } + + ret = cdns3_prepare_aligned_request_buf(priv_req); + if (ret < 0) + return ret; + + ret = usb_gadget_map_request(&priv_dev->gadget, request, + usb_endpoint_dir_in(ep->desc)); + if (ret) + return ret; + + list_add_tail(&request->list, &priv_ep->deferred_req_list); + + /* + * If hardware endpoint configuration has not been set yet then + * just queue request in deferred list. Transfer will be started in + * cdns3_set_hw_configuration. + */ + if (priv_dev->hw_configured_flag && !(priv_ep->flags & EP_STALLED) && + !(priv_ep->flags & EP_STALL_PENDING)) + cdns3_start_all_request(priv_dev, priv_ep); + + return 0; +} + +static int cdns3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request, + gfp_t gfp_flags) +{ + struct usb_request *zlp_request; + struct cdns3_endpoint *priv_ep; + struct cdns3_device *priv_dev; + unsigned long flags; + int ret; + + if (!request || !ep) + return -EINVAL; + + priv_ep = ep_to_cdns3_ep(ep); + priv_dev = priv_ep->cdns3_dev; + + spin_lock_irqsave(&priv_dev->lock, flags); + + ret = __cdns3_gadget_ep_queue(ep, request, gfp_flags); + + if (ret == 0 && request->zero && request->length && + (request->length % ep->maxpacket == 0)) { + struct cdns3_request *priv_req; + + zlp_request = cdns3_gadget_ep_alloc_request(ep, GFP_ATOMIC); + zlp_request->buf = priv_dev->zlp_buf; + zlp_request->length = 0; + + priv_req = to_cdns3_request(zlp_request); + priv_req->flags |= REQUEST_ZLP; + + dev_dbg(priv_dev->dev, "Queuing ZLP for endpoint: %s\n", + priv_ep->name); + ret = __cdns3_gadget_ep_queue(ep, zlp_request, gfp_flags); + } + + spin_unlock_irqrestore(&priv_dev->lock, flags); + return ret; +} + +/** + * cdns3_gadget_ep_dequeue Remove request from transfer queue + * @ep: endpoint object associated with request + * @request: request object + * + * Returns 0 on success, error code elsewhere + */ +int cdns3_gadget_ep_dequeue(struct usb_ep *ep, + struct usb_request *request) +{ + struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep); + struct cdns3_device *priv_dev = priv_ep->cdns3_dev; + struct usb_request *req, *req_temp; + struct cdns3_request *priv_req; + struct cdns3_trb *link_trb; + unsigned long flags; + int ret = 0; + + if (!ep || !request || !ep->desc) + return -EINVAL; + + spin_lock_irqsave(&priv_dev->lock, flags); + + priv_req = to_cdns3_request(request); + + trace_cdns3_ep_dequeue(priv_req); + + cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress); + + list_for_each_entry_safe(req, req_temp, &priv_ep->pending_req_list, + list) { + if (request == req) + goto found; + } + + list_for_each_entry_safe(req, req_temp, &priv_ep->deferred_req_list, + list) { + if (request == req) + goto found; + } + + goto not_found; + +found: + + if (priv_ep->wa1_trb == priv_req->trb) + cdns3_wa1_restore_cycle_bit(priv_ep); + + link_trb = priv_req->trb; + cdns3_move_deq_to_next_trb(priv_req); + cdns3_gadget_giveback(priv_ep, priv_req, -ECONNRESET); + + /* Update ring */ + request = cdns3_next_request(&priv_ep->deferred_req_list); + if (request) { + priv_req = to_cdns3_request(request); + + link_trb->buffer = TRB_BUFFER(priv_ep->trb_pool_dma + + (priv_req->start_trb * TRB_SIZE)); + link_trb->control = (link_trb->control & TRB_CYCLE) | + TRB_TYPE(TRB_LINK) | TRB_CHAIN | TRB_TOGGLE; + } else { + priv_ep->flags |= EP_UPDATE_EP_TRBADDR; + } + +not_found: + spin_unlock_irqrestore(&priv_dev->lock, flags); + return ret; +} + +/** + * __cdns3_gadget_ep_set_halt Sets stall on selected endpoint + * Should be called after acquiring spin_lock and selecting ep + * @ep: endpoint object to set stall on. + */ +void __cdns3_gadget_ep_set_halt(struct cdns3_endpoint *priv_ep) +{ + struct cdns3_device *priv_dev = priv_ep->cdns3_dev; + + trace_cdns3_halt(priv_ep, 1, 0); + + if (!(priv_ep->flags & EP_STALLED)) { + u32 ep_sts_reg = readl(&priv_dev->regs->ep_sts); + + if (!(ep_sts_reg & EP_STS_DBUSY)) + cdns3_ep_stall_flush(priv_ep); + else + priv_ep->flags |= EP_STALL_PENDING; + } +} + +/** + * __cdns3_gadget_ep_clear_halt Clears stall on selected endpoint + * Should be called after acquiring spin_lock and selecting ep + * @ep: endpoint object to clear stall on + */ +int __cdns3_gadget_ep_clear_halt(struct cdns3_endpoint *priv_ep) +{ + struct cdns3_device *priv_dev = priv_ep->cdns3_dev; + struct usb_request *request; + int ret = 0; + int val; + + trace_cdns3_halt(priv_ep, 0, 0); + + writel(EP_CMD_CSTALL | EP_CMD_EPRST, &priv_dev->regs->ep_cmd); + + /* wait for EPRST cleared */ + readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val, + !(val & EP_CMD_EPRST), 100); + if (ret) + return -EINVAL; + + priv_ep->flags &= ~(EP_STALLED | EP_STALL_PENDING); + + request = cdns3_next_request(&priv_ep->pending_req_list); + + if (request) + cdns3_rearm_transfer(priv_ep, 1); + + cdns3_start_all_request(priv_dev, priv_ep); + return ret; +} + +/** + * cdns3_gadget_ep_set_halt Sets/clears stall on selected endpoint + * @ep: endpoint object to set/clear stall on + * @value: 1 for set stall, 0 for clear stall + * + * Returns 0 on success, error code elsewhere + */ +int cdns3_gadget_ep_set_halt(struct usb_ep *ep, int value) +{ + struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep); + struct cdns3_device *priv_dev = priv_ep->cdns3_dev; + unsigned long flags; + int ret = 0; + + if (!(priv_ep->flags & EP_ENABLED)) + return -EPERM; + + spin_lock_irqsave(&priv_dev->lock, flags); + + cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress); + + if (!value) { + priv_ep->flags &= ~EP_WEDGE; + ret = __cdns3_gadget_ep_clear_halt(priv_ep); + } else { + __cdns3_gadget_ep_set_halt(priv_ep); + } + + spin_unlock_irqrestore(&priv_dev->lock, flags); + + return ret; +} + +extern const struct usb_ep_ops cdns3_gadget_ep0_ops; + +static const struct usb_ep_ops cdns3_gadget_ep_ops = { + .enable = cdns3_gadget_ep_enable, + .disable = cdns3_gadget_ep_disable, + .alloc_request = cdns3_gadget_ep_alloc_request, + .free_request = cdns3_gadget_ep_free_request, + .queue = cdns3_gadget_ep_queue, + .dequeue = cdns3_gadget_ep_dequeue, + .set_halt = cdns3_gadget_ep_set_halt, + .set_wedge = cdns3_gadget_ep_set_wedge, +}; + +/** + * cdns3_gadget_get_frame Returns number of actual ITP frame + * @gadget: gadget object + * + * Returns number of actual ITP frame + */ +static int cdns3_gadget_get_frame(struct usb_gadget *gadget) +{ + struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget); + + return readl(&priv_dev->regs->usb_itpn); +} + +int __cdns3_gadget_wakeup(struct cdns3_device *priv_dev) +{ + enum usb_device_speed speed; + + speed = cdns3_get_speed(priv_dev); + + if (speed >= USB_SPEED_SUPER) + return 0; + + /* Start driving resume signaling to indicate remote wakeup. */ + writel(USB_CONF_LGO_L0, &priv_dev->regs->usb_conf); + + return 0; +} + +static int cdns3_gadget_wakeup(struct usb_gadget *gadget) +{ + struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget); + unsigned long flags; + int ret = 0; + + spin_lock_irqsave(&priv_dev->lock, flags); + ret = __cdns3_gadget_wakeup(priv_dev); + spin_unlock_irqrestore(&priv_dev->lock, flags); + return ret; +} + +static int cdns3_gadget_set_selfpowered(struct usb_gadget *gadget, + int is_selfpowered) +{ + struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget); + unsigned long flags; + + spin_lock_irqsave(&priv_dev->lock, flags); + priv_dev->is_selfpowered = !!is_selfpowered; + spin_unlock_irqrestore(&priv_dev->lock, flags); + return 0; +} + +static int cdns3_gadget_pullup(struct usb_gadget *gadget, int is_on) +{ + struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget); + + if (is_on) + writel(USB_CONF_DEVEN, &priv_dev->regs->usb_conf); + else + writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf); + + return 0; +} + +static void cdns3_gadget_config(struct cdns3_device *priv_dev) +{ + struct cdns3_usb_regs __iomem *regs = priv_dev->regs; + u32 reg; + + cdns3_ep0_config(priv_dev); + + /* enable interrupts for endpoint 0 (in and out) */ + writel(EP_IEN_EP_OUT0 | EP_IEN_EP_IN0, ®s->ep_ien); + + /* + * Driver needs to modify LFPS minimal U1 Exit time for DEV_VER_TI_V1 + * revision of controller. + */ + if (priv_dev->dev_ver == DEV_VER_TI_V1) { + reg = readl(®s->dbg_link1); + + reg &= ~DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_MASK; + reg |= DBG_LINK1_LFPS_MIN_GEN_U1_EXIT(0x55) | + DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_SET; + writel(reg, ®s->dbg_link1); + } + + /* + * By default some platforms has set protected access to memory. + * This cause problem with cache, so driver restore non-secure + * access to memory. + */ + reg = readl(®s->dma_axi_ctrl); + reg |= DMA_AXI_CTRL_MARPROT(DMA_AXI_CTRL_NON_SECURE) | + DMA_AXI_CTRL_MAWPROT(DMA_AXI_CTRL_NON_SECURE); + writel(reg, ®s->dma_axi_ctrl); + + /* enable generic interrupt*/ + writel(USB_IEN_INIT, ®s->usb_ien); + writel(USB_CONF_CLK2OFFDS | USB_CONF_L1DS, ®s->usb_conf); + + cdns3_configure_dmult(priv_dev, NULL); + + cdns3_gadget_pullup(&priv_dev->gadget, 1); +} + +/** + * cdns3_gadget_udc_start Gadget start + * @gadget: gadget object + * @driver: driver which operates on this gadget + * + * Returns 0 on success, error code elsewhere + */ +static int cdns3_gadget_udc_start(struct usb_gadget *gadget, + struct usb_gadget_driver *driver) +{ + struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget); + unsigned long flags; + + spin_lock_irqsave(&priv_dev->lock, flags); + priv_dev->gadget_driver = driver; + cdns3_gadget_config(priv_dev); + spin_unlock_irqrestore(&priv_dev->lock, flags); + return 0; +} + +/** + * cdns3_gadget_udc_stop Stops gadget + * @gadget: gadget object + * + * Returns 0 + */ +static int cdns3_gadget_udc_stop(struct usb_gadget *gadget) +{ + struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget); + struct cdns3_endpoint *priv_ep; + u32 bEndpointAddress; + struct usb_ep *ep; + int ret = 0; + int val; + + priv_dev->gadget_driver = NULL; + + priv_dev->onchip_used_size = 0; + priv_dev->out_mem_is_allocated = 0; + priv_dev->gadget.speed = USB_SPEED_UNKNOWN; + + list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) { + priv_ep = ep_to_cdns3_ep(ep); + bEndpointAddress = priv_ep->num | priv_ep->dir; + cdns3_select_ep(priv_dev, bEndpointAddress); + writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd); + readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val, + !(val & EP_CMD_EPRST), 100); + } + + /* disable interrupt for device */ + writel(0, &priv_dev->regs->usb_ien); + writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf); + + return ret; +} + +static void cdns3_gadget_udc_set_speed(struct usb_gadget *gadget, + enum usb_device_speed speed) +{ + struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget); + + switch (speed) { + case USB_SPEED_FULL: + writel(USB_CONF_SFORCE_FS, &priv_dev->regs->usb_conf); + writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf); + break; + case USB_SPEED_HIGH: + writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf); + break; + case USB_SPEED_SUPER: + break; + default: + dev_err(cdns->dev, "invalid speed parameter %d\n", + speed); + } + + priv_dev->gadget.speed = speed; +} + +static const struct usb_gadget_ops cdns3_gadget_ops = { + .get_frame = cdns3_gadget_get_frame, + .wakeup = cdns3_gadget_wakeup, + .set_selfpowered = cdns3_gadget_set_selfpowered, + .pullup = cdns3_gadget_pullup, + .udc_start = cdns3_gadget_udc_start, + .udc_stop = cdns3_gadget_udc_stop, + .match_ep = cdns3_gadget_match_ep, + .udc_set_speed = cdns3_gadget_udc_set_speed, +}; + +static void cdns3_free_all_eps(struct cdns3_device *priv_dev) +{ + int i; + + /* ep0 OUT point to ep0 IN. */ + priv_dev->eps[16] = NULL; + + for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++) + if (priv_dev->eps[i]) { + cdns3_free_trb_pool(priv_dev->eps[i]); + devm_kfree(priv_dev->dev, priv_dev->eps[i]); + } +} + +/** + * cdns3_init_eps Initializes software endpoints of gadget + * @cdns3: extended gadget object + * + * Returns 0 on success, error code elsewhere + */ +static int cdns3_init_eps(struct cdns3_device *priv_dev) +{ + u32 ep_enabled_reg, iso_ep_reg; + struct cdns3_endpoint *priv_ep; + int ep_dir, ep_number; + u32 ep_mask; + int ret = 0; + int i; + + /* Read it from USB_CAP3 to USB_CAP5 */ + ep_enabled_reg = readl(&priv_dev->regs->usb_cap3); + iso_ep_reg = readl(&priv_dev->regs->usb_cap4); + + dev_dbg(priv_dev->dev, "Initializing non-zero endpoints\n"); + + for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++) { + ep_dir = i >> 4; /* i div 16 */ + ep_number = i & 0xF; /* i % 16 */ + ep_mask = BIT(i); + + if (!(ep_enabled_reg & ep_mask)) + continue; + + if (ep_dir && !ep_number) { + priv_dev->eps[i] = priv_dev->eps[0]; + continue; + } + + priv_ep = devm_kzalloc(priv_dev->dev, sizeof(*priv_ep), + GFP_KERNEL); + if (!priv_ep) { + ret = -ENOMEM; + goto err; + } + + /* set parent of endpoint object */ + priv_ep->cdns3_dev = priv_dev; + priv_dev->eps[i] = priv_ep; + priv_ep->num = ep_number; + priv_ep->dir = ep_dir ? USB_DIR_IN : USB_DIR_OUT; + + if (!ep_number) { + ret = cdns3_init_ep0(priv_dev, priv_ep); + if (ret) { + dev_err(priv_dev->dev, "Failed to init ep0\n"); + goto err; + } + } else { + snprintf(priv_ep->name, sizeof(priv_ep->name), "ep%d%s", + ep_number, !!ep_dir ? "in" : "out"); + priv_ep->endpoint.name = priv_ep->name; + + usb_ep_set_maxpacket_limit(&priv_ep->endpoint, + CDNS3_EP_MAX_PACKET_LIMIT); + priv_ep->endpoint.max_streams = CDNS3_EP_MAX_STREAMS; + priv_ep->endpoint.ops = &cdns3_gadget_ep_ops; + if (ep_dir) + priv_ep->endpoint.caps.dir_in = 1; + else + priv_ep->endpoint.caps.dir_out = 1; + + if (iso_ep_reg & ep_mask) + priv_ep->endpoint.caps.type_iso = 1; + + priv_ep->endpoint.caps.type_bulk = 1; + priv_ep->endpoint.caps.type_int = 1; + + list_add_tail(&priv_ep->endpoint.ep_list, + &priv_dev->gadget.ep_list); + } + + priv_ep->flags = 0; + + dev_info(priv_dev->dev, "Initialized %s support: %s %s\n", + priv_ep->name, + priv_ep->endpoint.caps.type_bulk ? "BULK, INT" : "", + priv_ep->endpoint.caps.type_iso ? "ISO" : ""); + + INIT_LIST_HEAD(&priv_ep->pending_req_list); + INIT_LIST_HEAD(&priv_ep->deferred_req_list); + INIT_LIST_HEAD(&priv_ep->wa2_descmiss_req_list); + } + + return 0; +err: + cdns3_free_all_eps(priv_dev); + return -ENOMEM; +} + +void cdns3_gadget_exit(struct cdns3 *cdns) +{ + struct cdns3_device *priv_dev; + + priv_dev = cdns->gadget_dev; + + usb_del_gadget_udc(&priv_dev->gadget); + + cdns3_free_all_eps(priv_dev); + + while (!list_empty(&priv_dev->aligned_buf_list)) { + struct cdns3_aligned_buf *buf; + + buf = cdns3_next_align_buf(&priv_dev->aligned_buf_list); + dma_free_coherent(buf->buf); + + list_del(&buf->list); + kfree(buf); + } + + dma_free_coherent(priv_dev->setup_buf); + + kfree(priv_dev->zlp_buf); + kfree(priv_dev); + cdns->gadget_dev = NULL; + cdns3_drd_switch_gadget(cdns, 0); +} + +static int cdns3_gadget_start(struct cdns3 *cdns) +{ + struct cdns3_device *priv_dev; + u32 max_speed; + int ret; + + priv_dev = kzalloc(sizeof(*priv_dev), GFP_KERNEL); + if (!priv_dev) + return -ENOMEM; + + cdns->gadget_dev = priv_dev; + priv_dev->sysdev = cdns->dev; + priv_dev->dev = cdns->dev; + priv_dev->regs = cdns->dev_regs; + + dev_read_u32(priv_dev->dev, "cdns,on-chip-buff-size", + &priv_dev->onchip_buffers); + + if (priv_dev->onchip_buffers <= 0) { + u32 reg = readl(&priv_dev->regs->usb_cap2); + + priv_dev->onchip_buffers = USB_CAP2_ACTUAL_MEM_SIZE(reg); + } + + if (!priv_dev->onchip_buffers) + priv_dev->onchip_buffers = 256; + + max_speed = usb_get_maximum_speed(dev_of_offset(cdns->dev)); + + /* Check the maximum_speed parameter */ + switch (max_speed) { + case USB_SPEED_FULL: + /* fall through */ + case USB_SPEED_HIGH: + /* fall through */ + case USB_SPEED_SUPER: + break; + default: + dev_err(cdns->dev, "invalid maximum_speed parameter %d\n", + max_speed); + /* fall through */ + case USB_SPEED_UNKNOWN: + /* default to superspeed */ + max_speed = USB_SPEED_SUPER; + break; + } + + /* fill gadget fields */ + priv_dev->gadget.max_speed = max_speed; + priv_dev->gadget.speed = USB_SPEED_UNKNOWN; + priv_dev->gadget.ops = &cdns3_gadget_ops; + priv_dev->gadget.name = "cdns3-gadget"; +#ifndef __UBOOT__ + priv_dev->gadget.name = "usb-ss-gadget"; + priv_dev->gadget.sg_supported = 1; + priv_dev->gadget.quirk_avoids_skb_reserve = 1; +#endif + + spin_lock_init(&priv_dev->lock); + INIT_WORK(&priv_dev->pending_status_wq, + cdns3_pending_setup_status_handler); + + /* initialize endpoint container */ + INIT_LIST_HEAD(&priv_dev->gadget.ep_list); + INIT_LIST_HEAD(&priv_dev->aligned_buf_list); + + ret = cdns3_init_eps(priv_dev); + if (ret) { + dev_err(priv_dev->dev, "Failed to create endpoints\n"); + goto err1; + } + + /* allocate memory for setup packet buffer */ + priv_dev->setup_buf = + dma_alloc_coherent(8, (unsigned long *)&priv_dev->setup_dma); + if (!priv_dev->setup_buf) { + ret = -ENOMEM; + goto err2; + } + + priv_dev->dev_ver = readl(&priv_dev->regs->usb_cap6); + + dev_dbg(priv_dev->dev, "Device Controller version: %08x\n", + readl(&priv_dev->regs->usb_cap6)); + dev_dbg(priv_dev->dev, "USB Capabilities:: %08x\n", + readl(&priv_dev->regs->usb_cap1)); + dev_dbg(priv_dev->dev, "On-Chip memory cnfiguration: %08x\n", + readl(&priv_dev->regs->usb_cap2)); + + priv_dev->dev_ver = GET_DEV_BASE_VERSION(priv_dev->dev_ver); + + priv_dev->zlp_buf = kzalloc(CDNS3_EP_ZLP_BUF_SIZE, GFP_KERNEL); + if (!priv_dev->zlp_buf) { + ret = -ENOMEM; + goto err3; + } + + /* add USB gadget device */ + ret = usb_add_gadget_udc((struct device *)priv_dev->dev, + &priv_dev->gadget); + if (ret < 0) { + dev_err(priv_dev->dev, + "Failed to register USB device controller\n"); + goto err4; + } + + return 0; +err4: + kfree(priv_dev->zlp_buf); +err3: + dma_free_coherent(priv_dev->setup_buf); +err2: + cdns3_free_all_eps(priv_dev); +err1: + cdns->gadget_dev = NULL; + return ret; +} + +static int __cdns3_gadget_init(struct cdns3 *cdns) +{ + int ret = 0; + + cdns3_drd_switch_gadget(cdns, 1); + + ret = cdns3_gadget_start(cdns); + if (ret) + return ret; + + return 0; +} + +static int cdns3_gadget_suspend(struct cdns3 *cdns, bool do_wakeup) +{ + struct cdns3_device *priv_dev = cdns->gadget_dev; + + cdns3_disconnect_gadget(priv_dev); + + priv_dev->gadget.speed = USB_SPEED_UNKNOWN; + usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED); + cdns3_hw_reset_eps_config(priv_dev); + + /* disable interrupt for device */ + writel(0, &priv_dev->regs->usb_ien); + + cdns3_gadget_pullup(&priv_dev->gadget, 0); + + return 0; +} + +static int cdns3_gadget_resume(struct cdns3 *cdns, bool hibernated) +{ + struct cdns3_device *priv_dev = cdns->gadget_dev; + + if (!priv_dev->gadget_driver) + return 0; + + cdns3_gadget_config(priv_dev); + + return 0; +} + +/** + * cdns3_gadget_init - initialize device structure + * + * cdns: cdns3 instance + * + * This function initializes the gadget. + */ +int cdns3_gadget_init(struct cdns3 *cdns) +{ + struct cdns3_role_driver *rdrv; + + rdrv = devm_kzalloc(cdns->dev, sizeof(*rdrv), GFP_KERNEL); + if (!rdrv) + return -ENOMEM; + + rdrv->start = __cdns3_gadget_init; + rdrv->stop = cdns3_gadget_exit; + rdrv->suspend = cdns3_gadget_suspend; + rdrv->resume = cdns3_gadget_resume; + rdrv->state = CDNS3_ROLE_STATE_INACTIVE; + rdrv->name = "gadget"; + cdns->roles[USB_ROLE_DEVICE] = rdrv; + + return 0; +} + +/** + * cdns3_gadget_uboot_handle_interrupt - handle cdns3 gadget interrupt + * @cdns: pointer to struct cdns3 + * + * Handles ep0 and gadget interrupt + */ +static void cdns3_gadget_uboot_handle_interrupt(struct cdns3 *cdns) +{ + int ret = cdns3_device_irq_handler(0, cdns); + + if (ret == IRQ_WAKE_THREAD) + cdns3_device_thread_irq_handler(0, cdns); +} + +int dm_usb_gadget_handle_interrupts(struct udevice *dev) +{ + struct cdns3 *cdns = dev_get_priv(dev); + + cdns3_gadget_uboot_handle_interrupt(cdns); + + return 0; +} diff --git a/drivers/usb/dwc3/core.c b/drivers/usb/dwc3/core.c index 2d64e991a9..178918c580 100644 --- a/drivers/usb/dwc3/core.c +++ b/drivers/usb/dwc3/core.c @@ -16,7 +16,8 @@ #include #include #include -#include +#include +#include #include #include #include diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c index 085f7b8968..4ec76a803c 100644 --- a/drivers/usb/dwc3/gadget.c +++ b/drivers/usb/dwc3/gadget.c @@ -15,8 +15,8 @@ #include #include -#include #include +#include #include #include diff --git a/drivers/usb/gadget/udc/udc-core.c b/drivers/usb/gadget/udc/udc-core.c index 06b18507ca..e6123ec4ab 100644 --- a/drivers/usb/gadget/udc/udc-core.c +++ b/drivers/usb/gadget/udc/udc-core.c @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h new file mode 100644 index 0000000000..20b6d60dd8 --- /dev/null +++ b/include/linux/dma-mapping.h @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_DMA_MAPPING_H +#define _LINUX_DMA_MAPPING_H + +#include +#include +#include +#include + +#define dma_mapping_error(x, y) 0 + +/** + * Map a buffer to make it available to the DMA device + * + * Linux-like DMA API that is intended to be used from drivers. This hides the + * underlying cache operation from drivers. Call this before starting the DMA + * transfer. In most of architectures in U-Boot, the virtual address matches to + * the physical address (but we have exceptions like sandbox). U-Boot does not + * support iommu at the driver level, so it also matches to the DMA address. + * Hence, this helper currently just performs the cache operation, then returns + * straight-mapped dma_address, which is intended to be set to the register of + * the DMA device. + * + * @vaddr: address of the buffer + * @len: length of the buffer + * @dir: the direction of DMA + */ +static inline dma_addr_t dma_map_single(void *vaddr, size_t len, + enum dma_data_direction dir) +{ + unsigned long addr = (unsigned long)vaddr; + + len = ALIGN(len, ARCH_DMA_MINALIGN); + + if (dir == DMA_FROM_DEVICE) + invalidate_dcache_range(addr, addr + len); + else + flush_dcache_range(addr, addr + len); + + return addr; +} + +/** + * Unmap a buffer to make it available to CPU + * + * Linux-like DMA API that is intended to be used from drivers. This hides the + * underlying cache operation from drivers. Call this after finishin the DMA + * transfer. + * + * @addr: DMA address + * @len: length of the buffer + * @dir: the direction of DMA + */ +static inline void dma_unmap_single(dma_addr_t addr, size_t len, + enum dma_data_direction dir) +{ + len = ALIGN(len, ARCH_DMA_MINALIGN); + + if (dir != DMA_TO_DEVICE) + invalidate_dcache_range(addr, addr + len); +} + +#endif