1 /* SPDX-License-Identifier: GPL-2.0-only */
3 * Low level PM code for TI EMIF
5 * Copyright (C) 2016-2017 Texas Instruments Incorporated - http://www.ti.com/
9 #include <linux/linkage.h>
10 #include <asm/assembler.h>
11 #include <asm/memory.h>
14 #include "ti-emif-asm-offsets.h"
16 #define EMIF_POWER_MGMT_WAIT_SELF_REFRESH_8192_CYCLES 0x00a0
17 #define EMIF_POWER_MGMT_SR_TIMER_MASK 0x00f0
18 #define EMIF_POWER_MGMT_SELF_REFRESH_MODE 0x0200
19 #define EMIF_POWER_MGMT_SELF_REFRESH_MODE_MASK 0x0700
21 #define EMIF_SDCFG_TYPE_DDR2 0x2 << SDRAM_TYPE_SHIFT
22 #define EMIF_SDCFG_TYPE_DDR3 0x3 << SDRAM_TYPE_SHIFT
23 #define EMIF_STATUS_READY 0x4
25 #define AM43XX_EMIF_PHY_CTRL_REG_COUNT 0x120
27 #define EMIF_AM437X_REGISTERS 0x1
35 * void ti_emif_save_context(void)
37 * Used during suspend to save the context of all required EMIF registers
38 * to local memory if the EMIF is going to lose context during the sleep
39 * transition. Operates on the VIRTUAL address of the EMIF.
41 ENTRY(ti_emif_save_context)
42 stmfd sp!, {r4 - r11, lr} @ save registers on stack
44 adr r4, ti_emif_pm_sram_data
45 ldr r0, [r4, #EMIF_PM_BASE_ADDR_VIRT_OFFSET]
46 ldr r2, [r4, #EMIF_PM_REGS_VIRT_OFFSET]
48 /* Save EMIF configuration */
49 ldr r1, [r0, #EMIF_SDRAM_CONFIG]
50 str r1, [r2, #EMIF_SDCFG_VAL_OFFSET]
52 ldr r1, [r0, #EMIF_SDRAM_REFRESH_CONTROL]
53 str r1, [r2, #EMIF_REF_CTRL_VAL_OFFSET]
55 ldr r1, [r0, #EMIF_SDRAM_TIMING_1]
56 str r1, [r2, #EMIF_TIMING1_VAL_OFFSET]
58 ldr r1, [r0, #EMIF_SDRAM_TIMING_2]
59 str r1, [r2, #EMIF_TIMING2_VAL_OFFSET]
61 ldr r1, [r0, #EMIF_SDRAM_TIMING_3]
62 str r1, [r2, #EMIF_TIMING3_VAL_OFFSET]
64 ldr r1, [r0, #EMIF_POWER_MANAGEMENT_CONTROL]
65 str r1, [r2, #EMIF_PMCR_VAL_OFFSET]
67 ldr r1, [r0, #EMIF_POWER_MANAGEMENT_CTRL_SHDW]
68 str r1, [r2, #EMIF_PMCR_SHDW_VAL_OFFSET]
70 ldr r1, [r0, #EMIF_SDRAM_OUTPUT_IMPEDANCE_CALIBRATION_CONFIG]
71 str r1, [r2, #EMIF_ZQCFG_VAL_OFFSET]
73 ldr r1, [r0, #EMIF_DDR_PHY_CTRL_1]
74 str r1, [r2, #EMIF_DDR_PHY_CTLR_1_OFFSET]
76 ldr r1, [r0, #EMIF_COS_CONFIG]
77 str r1, [r2, #EMIF_COS_CONFIG_OFFSET]
79 ldr r1, [r0, #EMIF_PRIORITY_TO_CLASS_OF_SERVICE_MAPPING]
80 str r1, [r2, #EMIF_PRIORITY_TO_COS_MAPPING_OFFSET]
82 ldr r1, [r0, #EMIF_CONNECTION_ID_TO_CLASS_OF_SERVICE_1_MAPPING]
83 str r1, [r2, #EMIF_CONNECT_ID_SERV_1_MAP_OFFSET]
85 ldr r1, [r0, #EMIF_CONNECTION_ID_TO_CLASS_OF_SERVICE_2_MAPPING]
86 str r1, [r2, #EMIF_CONNECT_ID_SERV_2_MAP_OFFSET]
88 ldr r1, [r0, #EMIF_OCP_CONFIG]
89 str r1, [r2, #EMIF_OCP_CONFIG_VAL_OFFSET]
91 ldr r5, [r4, #EMIF_PM_CONFIG_OFFSET]
92 cmp r5, #EMIF_SRAM_AM43_REG_LAYOUT
93 bne emif_skip_save_extra_regs
95 ldr r1, [r0, #EMIF_READ_WRITE_LEVELING_RAMP_CONTROL]
96 str r1, [r2, #EMIF_RD_WR_LEVEL_RAMP_CTRL_OFFSET]
98 ldr r1, [r0, #EMIF_READ_WRITE_EXECUTION_THRESHOLD]
99 str r1, [r2, #EMIF_RD_WR_EXEC_THRESH_OFFSET]
101 ldr r1, [r0, #EMIF_LPDDR2_NVM_TIMING]
102 str r1, [r2, #EMIF_LPDDR2_NVM_TIM_OFFSET]
104 ldr r1, [r0, #EMIF_LPDDR2_NVM_TIMING_SHDW]
105 str r1, [r2, #EMIF_LPDDR2_NVM_TIM_SHDW_OFFSET]
107 ldr r1, [r0, #EMIF_DLL_CALIB_CTRL]
108 str r1, [r2, #EMIF_DLL_CALIB_CTRL_VAL_OFFSET]
110 ldr r1, [r0, #EMIF_DLL_CALIB_CTRL_SHDW]
111 str r1, [r2, #EMIF_DLL_CALIB_CTRL_VAL_SHDW_OFFSET]
113 /* Loop and save entire block of emif phy regs */
115 add r4, r2, #EMIF_EXT_PHY_CTRL_VALS_OFFSET
116 add r3, r0, #EMIF_EXT_PHY_CTRL_1
121 cmp r5, #AM43XX_EMIF_PHY_CTRL_REG_COUNT
122 bne ddr_phy_ctrl_save
124 emif_skip_save_extra_regs:
125 ldmfd sp!, {r4 - r11, pc} @ restore regs and return
126 ENDPROC(ti_emif_save_context)
129 * void ti_emif_restore_context(void)
131 * Used during resume to restore the context of all required EMIF registers
132 * from local memory after the EMIF has lost context during a sleep transition.
133 * Operates on the PHYSICAL address of the EMIF.
135 ENTRY(ti_emif_restore_context)
136 adr r4, ti_emif_pm_sram_data
137 ldr r0, [r4, #EMIF_PM_BASE_ADDR_PHYS_OFFSET]
138 ldr r2, [r4, #EMIF_PM_REGS_PHYS_OFFSET]
140 /* Config EMIF Timings */
141 ldr r1, [r2, #EMIF_DDR_PHY_CTLR_1_OFFSET]
142 str r1, [r0, #EMIF_DDR_PHY_CTRL_1]
143 str r1, [r0, #EMIF_DDR_PHY_CTRL_1_SHDW]
145 ldr r1, [r2, #EMIF_TIMING1_VAL_OFFSET]
146 str r1, [r0, #EMIF_SDRAM_TIMING_1]
147 str r1, [r0, #EMIF_SDRAM_TIMING_1_SHDW]
149 ldr r1, [r2, #EMIF_TIMING2_VAL_OFFSET]
150 str r1, [r0, #EMIF_SDRAM_TIMING_2]
151 str r1, [r0, #EMIF_SDRAM_TIMING_2_SHDW]
153 ldr r1, [r2, #EMIF_TIMING3_VAL_OFFSET]
154 str r1, [r0, #EMIF_SDRAM_TIMING_3]
155 str r1, [r0, #EMIF_SDRAM_TIMING_3_SHDW]
157 ldr r1, [r2, #EMIF_REF_CTRL_VAL_OFFSET]
158 str r1, [r0, #EMIF_SDRAM_REFRESH_CONTROL]
159 str r1, [r0, #EMIF_SDRAM_REFRESH_CTRL_SHDW]
161 ldr r1, [r2, #EMIF_PMCR_VAL_OFFSET]
162 str r1, [r0, #EMIF_POWER_MANAGEMENT_CONTROL]
164 ldr r1, [r2, #EMIF_PMCR_SHDW_VAL_OFFSET]
165 str r1, [r0, #EMIF_POWER_MANAGEMENT_CTRL_SHDW]
167 ldr r1, [r2, #EMIF_COS_CONFIG_OFFSET]
168 str r1, [r0, #EMIF_COS_CONFIG]
170 ldr r1, [r2, #EMIF_PRIORITY_TO_COS_MAPPING_OFFSET]
171 str r1, [r0, #EMIF_PRIORITY_TO_CLASS_OF_SERVICE_MAPPING]
173 ldr r1, [r2, #EMIF_CONNECT_ID_SERV_1_MAP_OFFSET]
174 str r1, [r0, #EMIF_CONNECTION_ID_TO_CLASS_OF_SERVICE_1_MAPPING]
176 ldr r1, [r2, #EMIF_CONNECT_ID_SERV_2_MAP_OFFSET]
177 str r1, [r0, #EMIF_CONNECTION_ID_TO_CLASS_OF_SERVICE_2_MAPPING]
179 ldr r1, [r2, #EMIF_OCP_CONFIG_VAL_OFFSET]
180 str r1, [r0, #EMIF_OCP_CONFIG]
182 ldr r5, [r4, #EMIF_PM_CONFIG_OFFSET]
183 cmp r5, #EMIF_SRAM_AM43_REG_LAYOUT
184 bne emif_skip_restore_extra_regs
186 ldr r1, [r2, #EMIF_RD_WR_LEVEL_RAMP_CTRL_OFFSET]
187 str r1, [r0, #EMIF_READ_WRITE_LEVELING_RAMP_CONTROL]
189 ldr r1, [r2, #EMIF_RD_WR_EXEC_THRESH_OFFSET]
190 str r1, [r0, #EMIF_READ_WRITE_EXECUTION_THRESHOLD]
192 ldr r1, [r2, #EMIF_LPDDR2_NVM_TIM_OFFSET]
193 str r1, [r0, #EMIF_LPDDR2_NVM_TIMING]
195 ldr r1, [r2, #EMIF_LPDDR2_NVM_TIM_SHDW_OFFSET]
196 str r1, [r0, #EMIF_LPDDR2_NVM_TIMING_SHDW]
198 ldr r1, [r2, #EMIF_DLL_CALIB_CTRL_VAL_OFFSET]
199 str r1, [r0, #EMIF_DLL_CALIB_CTRL]
201 ldr r1, [r2, #EMIF_DLL_CALIB_CTRL_VAL_SHDW_OFFSET]
202 str r1, [r0, #EMIF_DLL_CALIB_CTRL_SHDW]
204 ldr r1, [r2, #EMIF_ZQCFG_VAL_OFFSET]
205 str r1, [r0, #EMIF_SDRAM_OUTPUT_IMPEDANCE_CALIBRATION_CONFIG]
207 /* Loop and restore entire block of emif phy regs */
209 /* Load ti_emif_regs_amx3 + EMIF_EXT_PHY_CTRL_VALS_OFFSET for address
210 * to phy register save space
212 add r3, r2, #EMIF_EXT_PHY_CTRL_VALS_OFFSET
213 add r4, r0, #EMIF_EXT_PHY_CTRL_1
214 ddr_phy_ctrl_restore:
218 cmp r5, #AM43XX_EMIF_PHY_CTRL_REG_COUNT
219 bne ddr_phy_ctrl_restore
221 emif_skip_restore_extra_regs:
223 * Output impedence calib needed only for DDR3
224 * but since the initial state of this will be
225 * disabled for DDR2 no harm in restoring the
228 ldr r1, [r2, #EMIF_ZQCFG_VAL_OFFSET]
229 str r1, [r0, #EMIF_SDRAM_OUTPUT_IMPEDANCE_CALIBRATION_CONFIG]
231 /* Write to sdcfg last for DDR2 only */
232 ldr r1, [r2, #EMIF_SDCFG_VAL_OFFSET]
233 and r2, r1, #SDRAM_TYPE_MASK
234 cmp r2, #EMIF_SDCFG_TYPE_DDR2
235 streq r1, [r0, #EMIF_SDRAM_CONFIG]
238 ENDPROC(ti_emif_restore_context)
241 * void ti_emif_run_hw_leveling(void)
243 * Used during resume to run hardware leveling again and restore the
244 * configuration of the EMIF PHY, only for DDR3.
246 ENTRY(ti_emif_run_hw_leveling)
247 adr r4, ti_emif_pm_sram_data
248 ldr r0, [r4, #EMIF_PM_BASE_ADDR_PHYS_OFFSET]
250 ldr r3, [r0, #EMIF_READ_WRITE_LEVELING_CONTROL]
251 orr r3, r3, #RDWRLVLFULL_START
252 ldr r2, [r0, #EMIF_SDRAM_CONFIG]
253 and r2, r2, #SDRAM_TYPE_MASK
254 cmp r2, #EMIF_SDCFG_TYPE_DDR3
257 str r3, [r0, #EMIF_READ_WRITE_LEVELING_CONTROL]
260 * If EMIF registers are touched during initial stage of HW
261 * leveling sequence there will be an L3 NOC timeout error issued
262 * as the EMIF will not respond, which is not fatal, but it is
263 * avoidable. This small wait loop is enough time for this condition
264 * to clear, even at worst case of CPU running at max speed of 1Ghz.
271 /* Bit clears when operation is complete */
272 2: ldr r1, [r0, #EMIF_READ_WRITE_LEVELING_CONTROL]
273 tst r1, #RDWRLVLFULL_START
278 ENDPROC(ti_emif_run_hw_leveling)
281 * void ti_emif_enter_sr(void)
283 * Programs the EMIF to tell the SDRAM to enter into self-refresh
284 * mode during a sleep transition. Operates on the VIRTUAL address
287 ENTRY(ti_emif_enter_sr)
288 stmfd sp!, {r4 - r11, lr} @ save registers on stack
290 adr r4, ti_emif_pm_sram_data
291 ldr r0, [r4, #EMIF_PM_BASE_ADDR_VIRT_OFFSET]
292 ldr r2, [r4, #EMIF_PM_REGS_VIRT_OFFSET]
294 ldr r1, [r0, #EMIF_POWER_MANAGEMENT_CONTROL]
295 bic r1, r1, #EMIF_POWER_MGMT_SELF_REFRESH_MODE_MASK
296 orr r1, r1, #EMIF_POWER_MGMT_SELF_REFRESH_MODE
297 str r1, [r0, #EMIF_POWER_MANAGEMENT_CONTROL]
299 ldmfd sp!, {r4 - r11, pc} @ restore regs and return
300 ENDPROC(ti_emif_enter_sr)
303 * void ti_emif_exit_sr(void)
305 * Programs the EMIF to tell the SDRAM to exit self-refresh mode
306 * after a sleep transition. Operates on the PHYSICAL address of
309 ENTRY(ti_emif_exit_sr)
310 adr r4, ti_emif_pm_sram_data
311 ldr r0, [r4, #EMIF_PM_BASE_ADDR_PHYS_OFFSET]
312 ldr r2, [r4, #EMIF_PM_REGS_PHYS_OFFSET]
315 * Toggle EMIF to exit refresh mode:
316 * if EMIF lost context, PWR_MGT_CTRL is currently 0, writing disable
317 * (0x0), wont do diddly squat! so do a toggle from SR(0x2) to disable
319 * *If* EMIF did not lose context, nothing broken as we write the same
320 * value(0x2) to reg before we write a disable (0x0).
322 ldr r1, [r2, #EMIF_PMCR_VAL_OFFSET]
323 bic r1, r1, #EMIF_POWER_MGMT_SELF_REFRESH_MODE_MASK
324 orr r1, r1, #EMIF_POWER_MGMT_SELF_REFRESH_MODE
325 str r1, [r0, #EMIF_POWER_MANAGEMENT_CONTROL]
326 bic r1, r1, #EMIF_POWER_MGMT_SELF_REFRESH_MODE_MASK
327 str r1, [r0, #EMIF_POWER_MANAGEMENT_CONTROL]
329 /* Wait for EMIF to become ready */
330 1: ldr r1, [r0, #EMIF_STATUS]
331 tst r1, #EMIF_STATUS_READY
335 ENDPROC(ti_emif_exit_sr)
338 * void ti_emif_abort_sr(void)
340 * Disables self-refresh after a failed transition to a low-power
341 * state so the kernel can jump back to DDR and follow abort path.
342 * Operates on the VIRTUAL address of the EMIF.
344 ENTRY(ti_emif_abort_sr)
345 stmfd sp!, {r4 - r11, lr} @ save registers on stack
347 adr r4, ti_emif_pm_sram_data
348 ldr r0, [r4, #EMIF_PM_BASE_ADDR_VIRT_OFFSET]
349 ldr r2, [r4, #EMIF_PM_REGS_VIRT_OFFSET]
351 ldr r1, [r2, #EMIF_PMCR_VAL_OFFSET]
352 bic r1, r1, #EMIF_POWER_MGMT_SELF_REFRESH_MODE_MASK
353 str r1, [r0, #EMIF_POWER_MANAGEMENT_CONTROL]
355 /* Wait for EMIF to become ready */
356 1: ldr r1, [r0, #EMIF_STATUS]
357 tst r1, #EMIF_STATUS_READY
360 ldmfd sp!, {r4 - r11, pc} @ restore regs and return
361 ENDPROC(ti_emif_abort_sr)
364 ENTRY(ti_emif_pm_sram_data)
365 .space EMIF_PM_DATA_SIZE
366 ENTRY(ti_emif_sram_sz)
367 .word . - ti_emif_save_context