2 * Copyright 2016 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
24 #include <linux/delay.h>
25 #include <linux/kernel.h>
26 #include <linux/firmware.h>
27 #include <linux/module.h>
28 #include <linux/pci.h>
31 #include "amdgpu_gfx.h"
34 #include "amdgpu_atomfirmware.h"
35 #include "amdgpu_pm.h"
37 #include "gc/gc_9_0_offset.h"
38 #include "gc/gc_9_0_sh_mask.h"
40 #include "vega10_enum.h"
41 #include "hdp/hdp_4_0_offset.h"
43 #include "soc15_common.h"
44 #include "clearstate_gfx9.h"
45 #include "v9_structs.h"
47 #include "ivsrcid/gfx/irqsrcs_gfx_9_0.h"
49 #include "amdgpu_ras.h"
53 #define GFX9_NUM_GFX_RINGS 1
54 #define GFX9_MEC_HPD_SIZE 4096
55 #define RLCG_UCODE_LOADING_START_ADDRESS 0x00002000L
56 #define RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET 0x00000000L
58 #define mmPWR_MISC_CNTL_STATUS 0x0183
59 #define mmPWR_MISC_CNTL_STATUS_BASE_IDX 0
60 #define PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN__SHIFT 0x0
61 #define PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS__SHIFT 0x1
62 #define PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK 0x00000001L
63 #define PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS_MASK 0x00000006L
65 #define mmGCEA_PROBE_MAP 0x070c
66 #define mmGCEA_PROBE_MAP_BASE_IDX 0
68 MODULE_FIRMWARE("amdgpu/vega10_ce.bin");
69 MODULE_FIRMWARE("amdgpu/vega10_pfp.bin");
70 MODULE_FIRMWARE("amdgpu/vega10_me.bin");
71 MODULE_FIRMWARE("amdgpu/vega10_mec.bin");
72 MODULE_FIRMWARE("amdgpu/vega10_mec2.bin");
73 MODULE_FIRMWARE("amdgpu/vega10_rlc.bin");
75 MODULE_FIRMWARE("amdgpu/vega12_ce.bin");
76 MODULE_FIRMWARE("amdgpu/vega12_pfp.bin");
77 MODULE_FIRMWARE("amdgpu/vega12_me.bin");
78 MODULE_FIRMWARE("amdgpu/vega12_mec.bin");
79 MODULE_FIRMWARE("amdgpu/vega12_mec2.bin");
80 MODULE_FIRMWARE("amdgpu/vega12_rlc.bin");
82 MODULE_FIRMWARE("amdgpu/vega20_ce.bin");
83 MODULE_FIRMWARE("amdgpu/vega20_pfp.bin");
84 MODULE_FIRMWARE("amdgpu/vega20_me.bin");
85 MODULE_FIRMWARE("amdgpu/vega20_mec.bin");
86 MODULE_FIRMWARE("amdgpu/vega20_mec2.bin");
87 MODULE_FIRMWARE("amdgpu/vega20_rlc.bin");
89 MODULE_FIRMWARE("amdgpu/raven_ce.bin");
90 MODULE_FIRMWARE("amdgpu/raven_pfp.bin");
91 MODULE_FIRMWARE("amdgpu/raven_me.bin");
92 MODULE_FIRMWARE("amdgpu/raven_mec.bin");
93 MODULE_FIRMWARE("amdgpu/raven_mec2.bin");
94 MODULE_FIRMWARE("amdgpu/raven_rlc.bin");
96 MODULE_FIRMWARE("amdgpu/picasso_ce.bin");
97 MODULE_FIRMWARE("amdgpu/picasso_pfp.bin");
98 MODULE_FIRMWARE("amdgpu/picasso_me.bin");
99 MODULE_FIRMWARE("amdgpu/picasso_mec.bin");
100 MODULE_FIRMWARE("amdgpu/picasso_mec2.bin");
101 MODULE_FIRMWARE("amdgpu/picasso_rlc.bin");
102 MODULE_FIRMWARE("amdgpu/picasso_rlc_am4.bin");
104 MODULE_FIRMWARE("amdgpu/raven2_ce.bin");
105 MODULE_FIRMWARE("amdgpu/raven2_pfp.bin");
106 MODULE_FIRMWARE("amdgpu/raven2_me.bin");
107 MODULE_FIRMWARE("amdgpu/raven2_mec.bin");
108 MODULE_FIRMWARE("amdgpu/raven2_mec2.bin");
109 MODULE_FIRMWARE("amdgpu/raven2_rlc.bin");
110 MODULE_FIRMWARE("amdgpu/raven_kicker_rlc.bin");
112 MODULE_FIRMWARE("amdgpu/arcturus_mec.bin");
113 MODULE_FIRMWARE("amdgpu/arcturus_mec2.bin");
114 MODULE_FIRMWARE("amdgpu/arcturus_rlc.bin");
116 MODULE_FIRMWARE("amdgpu/renoir_ce.bin");
117 MODULE_FIRMWARE("amdgpu/renoir_pfp.bin");
118 MODULE_FIRMWARE("amdgpu/renoir_me.bin");
119 MODULE_FIRMWARE("amdgpu/renoir_mec.bin");
120 MODULE_FIRMWARE("amdgpu/renoir_mec2.bin");
121 MODULE_FIRMWARE("amdgpu/renoir_rlc.bin");
123 #define mmTCP_CHAN_STEER_0_ARCT 0x0b03
124 #define mmTCP_CHAN_STEER_0_ARCT_BASE_IDX 0
125 #define mmTCP_CHAN_STEER_1_ARCT 0x0b04
126 #define mmTCP_CHAN_STEER_1_ARCT_BASE_IDX 0
127 #define mmTCP_CHAN_STEER_2_ARCT 0x0b09
128 #define mmTCP_CHAN_STEER_2_ARCT_BASE_IDX 0
129 #define mmTCP_CHAN_STEER_3_ARCT 0x0b0a
130 #define mmTCP_CHAN_STEER_3_ARCT_BASE_IDX 0
131 #define mmTCP_CHAN_STEER_4_ARCT 0x0b0b
132 #define mmTCP_CHAN_STEER_4_ARCT_BASE_IDX 0
133 #define mmTCP_CHAN_STEER_5_ARCT 0x0b0c
134 #define mmTCP_CHAN_STEER_5_ARCT_BASE_IDX 0
136 enum ta_ras_gfx_subblock {
138 TA_RAS_BLOCK__GFX_CPC_INDEX_START = 0,
139 TA_RAS_BLOCK__GFX_CPC_SCRATCH = TA_RAS_BLOCK__GFX_CPC_INDEX_START,
140 TA_RAS_BLOCK__GFX_CPC_UCODE,
141 TA_RAS_BLOCK__GFX_DC_STATE_ME1,
142 TA_RAS_BLOCK__GFX_DC_CSINVOC_ME1,
143 TA_RAS_BLOCK__GFX_DC_RESTORE_ME1,
144 TA_RAS_BLOCK__GFX_DC_STATE_ME2,
145 TA_RAS_BLOCK__GFX_DC_CSINVOC_ME2,
146 TA_RAS_BLOCK__GFX_DC_RESTORE_ME2,
147 TA_RAS_BLOCK__GFX_CPC_INDEX_END = TA_RAS_BLOCK__GFX_DC_RESTORE_ME2,
149 TA_RAS_BLOCK__GFX_CPF_INDEX_START,
150 TA_RAS_BLOCK__GFX_CPF_ROQ_ME2 = TA_RAS_BLOCK__GFX_CPF_INDEX_START,
151 TA_RAS_BLOCK__GFX_CPF_ROQ_ME1,
152 TA_RAS_BLOCK__GFX_CPF_TAG,
153 TA_RAS_BLOCK__GFX_CPF_INDEX_END = TA_RAS_BLOCK__GFX_CPF_TAG,
155 TA_RAS_BLOCK__GFX_CPG_INDEX_START,
156 TA_RAS_BLOCK__GFX_CPG_DMA_ROQ = TA_RAS_BLOCK__GFX_CPG_INDEX_START,
157 TA_RAS_BLOCK__GFX_CPG_DMA_TAG,
158 TA_RAS_BLOCK__GFX_CPG_TAG,
159 TA_RAS_BLOCK__GFX_CPG_INDEX_END = TA_RAS_BLOCK__GFX_CPG_TAG,
161 TA_RAS_BLOCK__GFX_GDS_INDEX_START,
162 TA_RAS_BLOCK__GFX_GDS_MEM = TA_RAS_BLOCK__GFX_GDS_INDEX_START,
163 TA_RAS_BLOCK__GFX_GDS_INPUT_QUEUE,
164 TA_RAS_BLOCK__GFX_GDS_OA_PHY_CMD_RAM_MEM,
165 TA_RAS_BLOCK__GFX_GDS_OA_PHY_DATA_RAM_MEM,
166 TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM,
167 TA_RAS_BLOCK__GFX_GDS_INDEX_END = TA_RAS_BLOCK__GFX_GDS_OA_PIPE_MEM,
169 TA_RAS_BLOCK__GFX_SPI_SR_MEM,
171 TA_RAS_BLOCK__GFX_SQ_INDEX_START,
172 TA_RAS_BLOCK__GFX_SQ_SGPR = TA_RAS_BLOCK__GFX_SQ_INDEX_START,
173 TA_RAS_BLOCK__GFX_SQ_LDS_D,
174 TA_RAS_BLOCK__GFX_SQ_LDS_I,
175 TA_RAS_BLOCK__GFX_SQ_VGPR, /* VGPR = SP*/
176 TA_RAS_BLOCK__GFX_SQ_INDEX_END = TA_RAS_BLOCK__GFX_SQ_VGPR,
178 TA_RAS_BLOCK__GFX_SQC_INDEX_START,
180 TA_RAS_BLOCK__GFX_SQC_INDEX0_START = TA_RAS_BLOCK__GFX_SQC_INDEX_START,
181 TA_RAS_BLOCK__GFX_SQC_INST_UTCL1_LFIFO =
182 TA_RAS_BLOCK__GFX_SQC_INDEX0_START,
183 TA_RAS_BLOCK__GFX_SQC_DATA_CU0_WRITE_DATA_BUF,
184 TA_RAS_BLOCK__GFX_SQC_DATA_CU0_UTCL1_LFIFO,
185 TA_RAS_BLOCK__GFX_SQC_DATA_CU1_WRITE_DATA_BUF,
186 TA_RAS_BLOCK__GFX_SQC_DATA_CU1_UTCL1_LFIFO,
187 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_WRITE_DATA_BUF,
188 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO,
189 TA_RAS_BLOCK__GFX_SQC_INDEX0_END =
190 TA_RAS_BLOCK__GFX_SQC_DATA_CU2_UTCL1_LFIFO,
192 TA_RAS_BLOCK__GFX_SQC_INDEX1_START,
193 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_TAG_RAM =
194 TA_RAS_BLOCK__GFX_SQC_INDEX1_START,
195 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO,
196 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_MISS_FIFO,
197 TA_RAS_BLOCK__GFX_SQC_INST_BANKA_BANK_RAM,
198 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_TAG_RAM,
199 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_HIT_FIFO,
200 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_MISS_FIFO,
201 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM,
202 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM,
203 TA_RAS_BLOCK__GFX_SQC_INDEX1_END =
204 TA_RAS_BLOCK__GFX_SQC_DATA_BANKA_BANK_RAM,
206 TA_RAS_BLOCK__GFX_SQC_INDEX2_START,
207 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_TAG_RAM =
208 TA_RAS_BLOCK__GFX_SQC_INDEX2_START,
209 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO,
210 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_MISS_FIFO,
211 TA_RAS_BLOCK__GFX_SQC_INST_BANKB_BANK_RAM,
212 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_TAG_RAM,
213 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_HIT_FIFO,
214 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_MISS_FIFO,
215 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM,
216 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM,
217 TA_RAS_BLOCK__GFX_SQC_INDEX2_END =
218 TA_RAS_BLOCK__GFX_SQC_DATA_BANKB_BANK_RAM,
219 TA_RAS_BLOCK__GFX_SQC_INDEX_END = TA_RAS_BLOCK__GFX_SQC_INDEX2_END,
221 TA_RAS_BLOCK__GFX_TA_INDEX_START,
222 TA_RAS_BLOCK__GFX_TA_FS_DFIFO = TA_RAS_BLOCK__GFX_TA_INDEX_START,
223 TA_RAS_BLOCK__GFX_TA_FS_AFIFO,
224 TA_RAS_BLOCK__GFX_TA_FL_LFIFO,
225 TA_RAS_BLOCK__GFX_TA_FX_LFIFO,
226 TA_RAS_BLOCK__GFX_TA_FS_CFIFO,
227 TA_RAS_BLOCK__GFX_TA_INDEX_END = TA_RAS_BLOCK__GFX_TA_FS_CFIFO,
229 TA_RAS_BLOCK__GFX_TCA_INDEX_START,
230 TA_RAS_BLOCK__GFX_TCA_HOLE_FIFO = TA_RAS_BLOCK__GFX_TCA_INDEX_START,
231 TA_RAS_BLOCK__GFX_TCA_REQ_FIFO,
232 TA_RAS_BLOCK__GFX_TCA_INDEX_END = TA_RAS_BLOCK__GFX_TCA_REQ_FIFO,
233 /* TCC (5 sub-ranges)*/
234 TA_RAS_BLOCK__GFX_TCC_INDEX_START,
236 TA_RAS_BLOCK__GFX_TCC_INDEX0_START = TA_RAS_BLOCK__GFX_TCC_INDEX_START,
237 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX0_START,
238 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_0_1,
239 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_0,
240 TA_RAS_BLOCK__GFX_TCC_CACHE_DATA_BANK_1_1,
241 TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_0,
242 TA_RAS_BLOCK__GFX_TCC_CACHE_DIRTY_BANK_1,
243 TA_RAS_BLOCK__GFX_TCC_HIGH_RATE_TAG,
244 TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG,
245 TA_RAS_BLOCK__GFX_TCC_INDEX0_END = TA_RAS_BLOCK__GFX_TCC_LOW_RATE_TAG,
247 TA_RAS_BLOCK__GFX_TCC_INDEX1_START,
248 TA_RAS_BLOCK__GFX_TCC_IN_USE_DEC = TA_RAS_BLOCK__GFX_TCC_INDEX1_START,
249 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER,
250 TA_RAS_BLOCK__GFX_TCC_INDEX1_END =
251 TA_RAS_BLOCK__GFX_TCC_IN_USE_TRANSFER,
253 TA_RAS_BLOCK__GFX_TCC_INDEX2_START,
254 TA_RAS_BLOCK__GFX_TCC_RETURN_DATA = TA_RAS_BLOCK__GFX_TCC_INDEX2_START,
255 TA_RAS_BLOCK__GFX_TCC_RETURN_CONTROL,
256 TA_RAS_BLOCK__GFX_TCC_UC_ATOMIC_FIFO,
257 TA_RAS_BLOCK__GFX_TCC_WRITE_RETURN,
258 TA_RAS_BLOCK__GFX_TCC_WRITE_CACHE_READ,
259 TA_RAS_BLOCK__GFX_TCC_SRC_FIFO,
260 TA_RAS_BLOCK__GFX_TCC_SRC_FIFO_NEXT_RAM,
261 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO,
262 TA_RAS_BLOCK__GFX_TCC_INDEX2_END =
263 TA_RAS_BLOCK__GFX_TCC_CACHE_TAG_PROBE_FIFO,
265 TA_RAS_BLOCK__GFX_TCC_INDEX3_START,
266 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO = TA_RAS_BLOCK__GFX_TCC_INDEX3_START,
267 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM,
268 TA_RAS_BLOCK__GFX_TCC_INDEX3_END =
269 TA_RAS_BLOCK__GFX_TCC_LATENCY_FIFO_NEXT_RAM,
271 TA_RAS_BLOCK__GFX_TCC_INDEX4_START,
272 TA_RAS_BLOCK__GFX_TCC_WRRET_TAG_WRITE_RETURN =
273 TA_RAS_BLOCK__GFX_TCC_INDEX4_START,
274 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER,
275 TA_RAS_BLOCK__GFX_TCC_INDEX4_END =
276 TA_RAS_BLOCK__GFX_TCC_ATOMIC_RETURN_BUFFER,
277 TA_RAS_BLOCK__GFX_TCC_INDEX_END = TA_RAS_BLOCK__GFX_TCC_INDEX4_END,
279 TA_RAS_BLOCK__GFX_TCI_WRITE_RAM,
281 TA_RAS_BLOCK__GFX_TCP_INDEX_START,
282 TA_RAS_BLOCK__GFX_TCP_CACHE_RAM = TA_RAS_BLOCK__GFX_TCP_INDEX_START,
283 TA_RAS_BLOCK__GFX_TCP_LFIFO_RAM,
284 TA_RAS_BLOCK__GFX_TCP_CMD_FIFO,
285 TA_RAS_BLOCK__GFX_TCP_VM_FIFO,
286 TA_RAS_BLOCK__GFX_TCP_DB_RAM,
287 TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO0,
288 TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1,
289 TA_RAS_BLOCK__GFX_TCP_INDEX_END = TA_RAS_BLOCK__GFX_TCP_UTCL1_LFIFO1,
291 TA_RAS_BLOCK__GFX_TD_INDEX_START,
292 TA_RAS_BLOCK__GFX_TD_SS_FIFO_LO = TA_RAS_BLOCK__GFX_TD_INDEX_START,
293 TA_RAS_BLOCK__GFX_TD_SS_FIFO_HI,
294 TA_RAS_BLOCK__GFX_TD_CS_FIFO,
295 TA_RAS_BLOCK__GFX_TD_INDEX_END = TA_RAS_BLOCK__GFX_TD_CS_FIFO,
296 /* EA (3 sub-ranges)*/
297 TA_RAS_BLOCK__GFX_EA_INDEX_START,
299 TA_RAS_BLOCK__GFX_EA_INDEX0_START = TA_RAS_BLOCK__GFX_EA_INDEX_START,
300 TA_RAS_BLOCK__GFX_EA_DRAMRD_CMDMEM = TA_RAS_BLOCK__GFX_EA_INDEX0_START,
301 TA_RAS_BLOCK__GFX_EA_DRAMWR_CMDMEM,
302 TA_RAS_BLOCK__GFX_EA_DRAMWR_DATAMEM,
303 TA_RAS_BLOCK__GFX_EA_RRET_TAGMEM,
304 TA_RAS_BLOCK__GFX_EA_WRET_TAGMEM,
305 TA_RAS_BLOCK__GFX_EA_GMIRD_CMDMEM,
306 TA_RAS_BLOCK__GFX_EA_GMIWR_CMDMEM,
307 TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM,
308 TA_RAS_BLOCK__GFX_EA_INDEX0_END = TA_RAS_BLOCK__GFX_EA_GMIWR_DATAMEM,
310 TA_RAS_BLOCK__GFX_EA_INDEX1_START,
311 TA_RAS_BLOCK__GFX_EA_DRAMRD_PAGEMEM = TA_RAS_BLOCK__GFX_EA_INDEX1_START,
312 TA_RAS_BLOCK__GFX_EA_DRAMWR_PAGEMEM,
313 TA_RAS_BLOCK__GFX_EA_IORD_CMDMEM,
314 TA_RAS_BLOCK__GFX_EA_IOWR_CMDMEM,
315 TA_RAS_BLOCK__GFX_EA_IOWR_DATAMEM,
316 TA_RAS_BLOCK__GFX_EA_GMIRD_PAGEMEM,
317 TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM,
318 TA_RAS_BLOCK__GFX_EA_INDEX1_END = TA_RAS_BLOCK__GFX_EA_GMIWR_PAGEMEM,
320 TA_RAS_BLOCK__GFX_EA_INDEX2_START,
321 TA_RAS_BLOCK__GFX_EA_MAM_D0MEM = TA_RAS_BLOCK__GFX_EA_INDEX2_START,
322 TA_RAS_BLOCK__GFX_EA_MAM_D1MEM,
323 TA_RAS_BLOCK__GFX_EA_MAM_D2MEM,
324 TA_RAS_BLOCK__GFX_EA_MAM_D3MEM,
325 TA_RAS_BLOCK__GFX_EA_INDEX2_END = TA_RAS_BLOCK__GFX_EA_MAM_D3MEM,
326 TA_RAS_BLOCK__GFX_EA_INDEX_END = TA_RAS_BLOCK__GFX_EA_INDEX2_END,
328 TA_RAS_BLOCK__UTC_VML2_BANK_CACHE,
330 TA_RAS_BLOCK__UTC_VML2_WALKER,
331 /* UTC ATC L2 2MB cache*/
332 TA_RAS_BLOCK__UTC_ATCL2_CACHE_2M_BANK,
333 /* UTC ATC L2 4KB cache*/
334 TA_RAS_BLOCK__UTC_ATCL2_CACHE_4K_BANK,
335 TA_RAS_BLOCK__GFX_MAX
338 struct ras_gfx_subblock {
341 int hw_supported_error_type;
342 int sw_supported_error_type;
345 #define AMDGPU_RAS_SUB_BLOCK(subblock, a, b, c, d, e, f, g, h) \
346 [AMDGPU_RAS_BLOCK__##subblock] = { \
348 TA_RAS_BLOCK__##subblock, \
349 ((a) | ((b) << 1) | ((c) << 2) | ((d) << 3)), \
350 (((e) << 1) | ((f) << 3) | (g) | ((h) << 2)), \
353 static const struct ras_gfx_subblock ras_gfx_subblocks[] = {
354 AMDGPU_RAS_SUB_BLOCK(GFX_CPC_SCRATCH, 0, 1, 1, 1, 1, 0, 0, 1),
355 AMDGPU_RAS_SUB_BLOCK(GFX_CPC_UCODE, 0, 1, 1, 1, 1, 0, 0, 1),
356 AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME1, 1, 0, 0, 1, 0, 0, 1, 0),
357 AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME1, 1, 0, 0, 1, 0, 0, 0, 0),
358 AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME1, 1, 0, 0, 1, 0, 0, 0, 0),
359 AMDGPU_RAS_SUB_BLOCK(GFX_DC_STATE_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
360 AMDGPU_RAS_SUB_BLOCK(GFX_DC_CSINVOC_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
361 AMDGPU_RAS_SUB_BLOCK(GFX_DC_RESTORE_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
362 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME2, 1, 0, 0, 1, 0, 0, 0, 0),
363 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_ROQ_ME1, 1, 0, 0, 1, 0, 0, 1, 0),
364 AMDGPU_RAS_SUB_BLOCK(GFX_CPF_TAG, 0, 1, 1, 1, 1, 0, 0, 1),
365 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_ROQ, 1, 0, 0, 1, 0, 0, 1, 0),
366 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_DMA_TAG, 0, 1, 1, 1, 0, 1, 0, 1),
367 AMDGPU_RAS_SUB_BLOCK(GFX_CPG_TAG, 0, 1, 1, 1, 1, 1, 0, 1),
368 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_MEM, 0, 1, 1, 1, 0, 0, 0, 0),
369 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_INPUT_QUEUE, 1, 0, 0, 1, 0, 0, 0, 0),
370 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_CMD_RAM_MEM, 0, 1, 1, 1, 0, 0, 0,
372 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PHY_DATA_RAM_MEM, 1, 0, 0, 1, 0, 0, 0,
374 AMDGPU_RAS_SUB_BLOCK(GFX_GDS_OA_PIPE_MEM, 0, 1, 1, 1, 0, 0, 0, 0),
375 AMDGPU_RAS_SUB_BLOCK(GFX_SPI_SR_MEM, 1, 0, 0, 1, 0, 0, 0, 0),
376 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_SGPR, 0, 1, 1, 1, 0, 0, 0, 0),
377 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_D, 0, 1, 1, 1, 1, 0, 0, 1),
378 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_LDS_I, 0, 1, 1, 1, 0, 0, 0, 0),
379 AMDGPU_RAS_SUB_BLOCK(GFX_SQ_VGPR, 0, 1, 1, 1, 0, 0, 0, 0),
380 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0, 1),
381 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
383 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU0_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0,
385 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
387 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU1_UTCL1_LFIFO, 0, 1, 1, 1, 1, 0, 0,
389 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_WRITE_DATA_BUF, 0, 1, 1, 1, 0, 0,
391 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_CU2_UTCL1_LFIFO, 0, 1, 1, 1, 0, 0, 0,
393 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_TAG_RAM, 0, 1, 1, 1, 1, 0, 0,
395 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0,
397 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
399 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
401 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_TAG_RAM, 0, 1, 1, 1, 0, 0, 0,
403 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0,
405 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
407 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0,
409 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKA_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
411 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_TAG_RAM, 0, 1, 1, 1, 1, 0, 0,
413 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_UTCL1_MISS_FIFO, 1, 0, 0, 1, 0,
415 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
417 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_INST_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
419 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_TAG_RAM, 0, 1, 1, 1, 0, 0, 0,
421 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_HIT_FIFO, 1, 0, 0, 1, 0, 0, 0,
423 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_MISS_FIFO, 1, 0, 0, 1, 0, 0, 0,
425 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_DIRTY_BIT_RAM, 1, 0, 0, 1, 0, 0,
427 AMDGPU_RAS_SUB_BLOCK(GFX_SQC_DATA_BANKB_BANK_RAM, 0, 1, 1, 1, 0, 0, 0,
429 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_DFIFO, 0, 1, 1, 1, 1, 0, 0, 1),
430 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_AFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
431 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FL_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
432 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FX_LFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
433 AMDGPU_RAS_SUB_BLOCK(GFX_TA_FS_CFIFO, 1, 0, 0, 1, 0, 0, 0, 0),
434 AMDGPU_RAS_SUB_BLOCK(GFX_TCA_HOLE_FIFO, 1, 0, 0, 1, 0, 1, 1, 0),
435 AMDGPU_RAS_SUB_BLOCK(GFX_TCA_REQ_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
436 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA, 0, 1, 1, 1, 1, 0, 0, 1),
437 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_0_1, 0, 1, 1, 1, 1, 0, 0,
439 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_0, 0, 1, 1, 1, 1, 0, 0,
441 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DATA_BANK_1_1, 0, 1, 1, 1, 1, 0, 0,
443 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_0, 0, 1, 1, 1, 0, 0, 0,
445 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_DIRTY_BANK_1, 0, 1, 1, 1, 0, 0, 0,
447 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_HIGH_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0),
448 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LOW_RATE_TAG, 0, 1, 1, 1, 0, 0, 0, 0),
449 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_DEC, 1, 0, 0, 1, 0, 0, 0, 0),
450 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_IN_USE_TRANSFER, 1, 0, 0, 1, 0, 0, 0, 0),
451 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_DATA, 1, 0, 0, 1, 0, 0, 0, 0),
452 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_RETURN_CONTROL, 1, 0, 0, 1, 0, 0, 0, 0),
453 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_UC_ATOMIC_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
454 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_RETURN, 1, 0, 0, 1, 0, 1, 1, 0),
455 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRITE_CACHE_READ, 1, 0, 0, 1, 0, 0, 0, 0),
456 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO, 0, 1, 1, 1, 0, 0, 0, 0),
457 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_SRC_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 1, 0),
458 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_CACHE_TAG_PROBE_FIFO, 1, 0, 0, 1, 0, 0, 0,
460 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
461 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_LATENCY_FIFO_NEXT_RAM, 1, 0, 0, 1, 0, 0, 0,
463 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_WRRET_TAG_WRITE_RETURN, 1, 0, 0, 1, 0, 0,
465 AMDGPU_RAS_SUB_BLOCK(GFX_TCC_ATOMIC_RETURN_BUFFER, 1, 0, 0, 1, 0, 0, 0,
467 AMDGPU_RAS_SUB_BLOCK(GFX_TCI_WRITE_RAM, 1, 0, 0, 1, 0, 0, 0, 0),
468 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CACHE_RAM, 0, 1, 1, 1, 1, 0, 0, 1),
469 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_LFIFO_RAM, 0, 1, 1, 1, 0, 0, 0, 0),
470 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_CMD_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
471 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_VM_FIFO, 0, 1, 1, 1, 0, 0, 0, 0),
472 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_DB_RAM, 1, 0, 0, 1, 0, 0, 0, 0),
473 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO0, 0, 1, 1, 1, 0, 0, 0, 0),
474 AMDGPU_RAS_SUB_BLOCK(GFX_TCP_UTCL1_LFIFO1, 0, 1, 1, 1, 0, 0, 0, 0),
475 AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_LO, 0, 1, 1, 1, 1, 0, 0, 1),
476 AMDGPU_RAS_SUB_BLOCK(GFX_TD_SS_FIFO_HI, 0, 1, 1, 1, 0, 0, 0, 0),
477 AMDGPU_RAS_SUB_BLOCK(GFX_TD_CS_FIFO, 1, 0, 0, 1, 0, 0, 0, 0),
478 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_CMDMEM, 0, 1, 1, 1, 1, 0, 0, 1),
479 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
480 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0),
481 AMDGPU_RAS_SUB_BLOCK(GFX_EA_RRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0),
482 AMDGPU_RAS_SUB_BLOCK(GFX_EA_WRET_TAGMEM, 0, 1, 1, 1, 0, 0, 0, 0),
483 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
484 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_CMDMEM, 0, 1, 1, 1, 0, 0, 0, 0),
485 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_DATAMEM, 0, 1, 1, 1, 0, 0, 0, 0),
486 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
487 AMDGPU_RAS_SUB_BLOCK(GFX_EA_DRAMWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
488 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IORD_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0),
489 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_CMDMEM, 1, 0, 0, 1, 0, 0, 0, 0),
490 AMDGPU_RAS_SUB_BLOCK(GFX_EA_IOWR_DATAMEM, 1, 0, 0, 1, 0, 0, 0, 0),
491 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIRD_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
492 AMDGPU_RAS_SUB_BLOCK(GFX_EA_GMIWR_PAGEMEM, 1, 0, 0, 1, 0, 0, 0, 0),
493 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D0MEM, 1, 0, 0, 1, 0, 0, 0, 0),
494 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D1MEM, 1, 0, 0, 1, 0, 0, 0, 0),
495 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D2MEM, 1, 0, 0, 1, 0, 0, 0, 0),
496 AMDGPU_RAS_SUB_BLOCK(GFX_EA_MAM_D3MEM, 1, 0, 0, 1, 0, 0, 0, 0),
497 AMDGPU_RAS_SUB_BLOCK(UTC_VML2_BANK_CACHE, 0, 1, 1, 1, 0, 0, 0, 0),
498 AMDGPU_RAS_SUB_BLOCK(UTC_VML2_WALKER, 0, 1, 1, 1, 0, 0, 0, 0),
499 AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_2M_BANK, 1, 0, 0, 1, 0, 0, 0, 0),
500 AMDGPU_RAS_SUB_BLOCK(UTC_ATCL2_CACHE_4K_BANK, 0, 1, 1, 1, 0, 0, 0, 0),
503 static const struct soc15_reg_golden golden_settings_gc_9_0[] =
505 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400),
506 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG3, 0x80000000, 0x80000000),
507 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
508 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
509 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
510 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
511 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
512 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),
513 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),
514 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x0000ff87),
515 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x0000ff8f),
516 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
517 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
518 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
519 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
520 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
521 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff),
522 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
523 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
524 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
527 static const struct soc15_reg_golden golden_settings_gc_9_0_vg10[] =
529 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0x0000f000, 0x00012107),
530 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
531 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
532 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
533 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
534 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
535 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x2a114042),
536 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
537 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0x00008000, 0x00048000),
538 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
539 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
540 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
541 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
542 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
543 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
544 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0x0000000f, 0x01000107),
545 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x00001800, 0x00000800),
546 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080)
549 static const struct soc15_reg_golden golden_settings_gc_9_0_vg20[] =
551 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x0f000080, 0x04000080),
552 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),
553 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
554 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x22014042),
555 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x22014042),
556 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0x00003e00, 0x00000400),
557 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xff840000, 0x04040000),
558 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00030000),
559 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff010f, 0x01000107),
560 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0x000b0000, 0x000b0000),
561 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01000000, 0x01000000)
564 static const struct soc15_reg_golden golden_settings_gc_9_1[] =
566 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
567 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x08000000, 0x08000080),
568 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0x08000000, 0x08000080),
569 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x08000000, 0x08000080),
570 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
571 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
572 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x08000000, 0x08000080),
573 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
574 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
575 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
576 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0x08000000, 0x08000080),
577 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0x08000000, 0x08000080),
578 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0x08000000, 0x08000080),
579 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0x08000000, 0x08000080),
580 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0x08000000, 0x08000080),
581 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
582 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
583 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),
584 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
585 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000000ff),
586 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x08000000, 0x08000080),
587 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
588 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
589 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
592 static const struct soc15_reg_golden golden_settings_gc_9_1_rv1[] =
594 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_3, 0x30000000, 0x10000000),
595 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24000042),
596 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24000042),
597 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04048000),
598 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_MODE_CNTL_1, 0x06000000, 0x06000000),
599 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRMI_UTCL1_CNTL2, 0x00030000, 0x00020000),
600 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x00000800)
603 static const struct soc15_reg_golden golden_settings_gc_9_1_rv2[] =
605 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0xff7fffff, 0x04000000),
606 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
607 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000),
608 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPC_UTCL1_CNTL, 0x7f0fffff, 0x08000080),
609 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPF_UTCL1_CNTL, 0xff8fffff, 0x08000080),
610 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCPG_UTCL1_CNTL, 0x7f8fffff, 0x08000080),
611 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x26013041),
612 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x26013041),
613 SOC15_REG_GOLDEN_VALUE(GC, 0, mmIA_UTCL1_CNTL, 0x3f8fffff, 0x08000080),
614 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
615 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_0, 0xff0fffff, 0x08000080),
616 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_1, 0xff0fffff, 0x08000080),
617 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_GPM_UTCL1_CNTL_2, 0xff0fffff, 0x08000080),
618 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_PREWALKER_UTCL1_CNTL, 0xff0fffff, 0x08000080),
619 SOC15_REG_GOLDEN_VALUE(GC, 0, mmRLC_SPM_UTCL1_CNTL, 0xff0fffff, 0x08000080),
620 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
621 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00000010),
622 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000),
623 SOC15_REG_GOLDEN_VALUE(GC, 0, mmWD_UTCL1_CNTL, 0x3f8fffff, 0x08000080),
626 static const struct soc15_reg_golden golden_settings_gc_9_1_rn[] =
628 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
629 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0xff7fffff, 0x0a000000),
630 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000400),
631 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xf3e777ff, 0x24000042),
632 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xf3e777ff, 0x24000042),
633 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
634 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
635 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
636 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
637 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
638 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x00003120),
639 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGCEA_PROBE_MAP, 0xffffffff, 0x0000cccc),
642 static const struct soc15_reg_golden golden_settings_gc_9_x_common[] =
644 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_SD_CNTL, 0xffffffff, 0x000001ff),
645 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_INDEX, 0xffffffff, 0x00000000),
646 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGRBM_CAM_DATA, 0xffffffff, 0x2544c382)
649 static const struct soc15_reg_golden golden_settings_gc_9_2_1[] =
651 SOC15_REG_GOLDEN_VALUE(GC, 0, mmDB_DEBUG2, 0xf00fffff, 0x00000420),
652 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_GPU_ID, 0x0000000f, 0x00000000),
653 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_BINNER_EVENT_CNTL_3, 0x00000003, 0x82400024),
654 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE, 0x3fffffff, 0x00000001),
655 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_LINE_STIPPLE_STATE, 0x0000ff0f, 0x00000000),
656 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSH_MEM_CONFIG, 0x00001000, 0x00001000),
657 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_0, 0x0007ffff, 0x00000800),
658 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_CU_1, 0x0007ffff, 0x00000800),
659 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_0, 0x01ffffff, 0x0000ff87),
660 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_RESOURCE_RESERVE_EN_CU_1, 0x01ffffff, 0x0000ff8f),
661 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQC_CONFIG, 0x03000000, 0x020a2000),
662 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x010b0000),
663 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x4a2c0e68),
664 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0xb5d3f197),
665 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_CACHE_INVALIDATION, 0x3fff3af3, 0x19200000),
666 SOC15_REG_GOLDEN_VALUE(GC, 0, mmVGT_GS_MAX_WAVE_ID, 0x00000fff, 0x000003ff)
669 static const struct soc15_reg_golden golden_settings_gc_9_2_1_vg12[] =
671 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_DCC_CONFIG, 0x00000080, 0x04000080),
672 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL, 0xfffdf3cf, 0x00014104),
673 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCB_HW_CONTROL_2, 0x0f000000, 0x0a000000),
674 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x24104041),
675 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG_READ, 0xffff77ff, 0x24104041),
676 SOC15_REG_GOLDEN_VALUE(GC, 0, mmPA_SC_ENHANCE_1, 0xffffffff, 0x04040000),
677 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSPI_CONFIG_CNTL_1, 0xffff03ff, 0x01000107),
678 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_HI, 0xffffffff, 0x00000000),
679 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_LO, 0xffffffff, 0x76325410),
680 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTD_CNTL, 0x01bd9f33, 0x01000000),
681 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC1_F32_INT_DIS, 0x00000800, 0x00000800),
682 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_MEC2_F32_INT_DIS, 0x00000800, 0x00000800),
683 SOC15_REG_GOLDEN_VALUE(GC, 0, mmCP_DEBUG, 0x00008000, 0x00008000)
686 static const struct soc15_reg_golden golden_settings_gc_9_4_1_arct[] =
688 SOC15_REG_GOLDEN_VALUE(GC, 0, mmGB_ADDR_CONFIG, 0xffff77ff, 0x2a114042),
689 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTA_CNTL_AUX, 0xfffffeef, 0x10b0000),
690 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_0_ARCT, 0x3fffffff, 0x346f0a4e),
691 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_1_ARCT, 0x3fffffff, 0x1c642ca),
692 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_2_ARCT, 0x3fffffff, 0x26f45098),
693 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_3_ARCT, 0x3fffffff, 0x2ebd9fe3),
694 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_4_ARCT, 0x3fffffff, 0xb90f5b1),
695 SOC15_REG_GOLDEN_VALUE(GC, 0, mmTCP_CHAN_STEER_5_ARCT, 0x3ff, 0x135),
696 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_CONFIG, 0xffffffff, 0x011A0000),
697 SOC15_REG_GOLDEN_VALUE(GC, 0, mmSQ_FIFO_SIZES, 0xffffffff, 0x00000f00),
700 static const struct soc15_reg_rlcg rlcg_access_gc_9_0[] = {
701 {SOC15_REG_ENTRY(GC, 0, mmGRBM_GFX_INDEX)},
702 {SOC15_REG_ENTRY(GC, 0, mmSQ_IND_INDEX)},
705 static const u32 GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[] =
707 mmRLC_SRM_INDEX_CNTL_ADDR_0 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
708 mmRLC_SRM_INDEX_CNTL_ADDR_1 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
709 mmRLC_SRM_INDEX_CNTL_ADDR_2 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
710 mmRLC_SRM_INDEX_CNTL_ADDR_3 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
711 mmRLC_SRM_INDEX_CNTL_ADDR_4 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
712 mmRLC_SRM_INDEX_CNTL_ADDR_5 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
713 mmRLC_SRM_INDEX_CNTL_ADDR_6 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
714 mmRLC_SRM_INDEX_CNTL_ADDR_7 - mmRLC_SRM_INDEX_CNTL_ADDR_0,
717 static const u32 GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[] =
719 mmRLC_SRM_INDEX_CNTL_DATA_0 - mmRLC_SRM_INDEX_CNTL_DATA_0,
720 mmRLC_SRM_INDEX_CNTL_DATA_1 - mmRLC_SRM_INDEX_CNTL_DATA_0,
721 mmRLC_SRM_INDEX_CNTL_DATA_2 - mmRLC_SRM_INDEX_CNTL_DATA_0,
722 mmRLC_SRM_INDEX_CNTL_DATA_3 - mmRLC_SRM_INDEX_CNTL_DATA_0,
723 mmRLC_SRM_INDEX_CNTL_DATA_4 - mmRLC_SRM_INDEX_CNTL_DATA_0,
724 mmRLC_SRM_INDEX_CNTL_DATA_5 - mmRLC_SRM_INDEX_CNTL_DATA_0,
725 mmRLC_SRM_INDEX_CNTL_DATA_6 - mmRLC_SRM_INDEX_CNTL_DATA_0,
726 mmRLC_SRM_INDEX_CNTL_DATA_7 - mmRLC_SRM_INDEX_CNTL_DATA_0,
729 void gfx_v9_0_rlcg_wreg(struct amdgpu_device *adev, u32 offset, u32 v)
731 static void *scratch_reg0;
732 static void *scratch_reg1;
733 static void *scratch_reg2;
734 static void *scratch_reg3;
735 static void *spare_int;
736 static uint32_t grbm_cntl;
737 static uint32_t grbm_idx;
739 scratch_reg0 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG0_BASE_IDX] + mmSCRATCH_REG0)*4;
740 scratch_reg1 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG1)*4;
741 scratch_reg2 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG2)*4;
742 scratch_reg3 = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmSCRATCH_REG1_BASE_IDX] + mmSCRATCH_REG3)*4;
743 spare_int = adev->rmmio + (adev->reg_offset[GC_HWIP][0][mmRLC_SPARE_INT_BASE_IDX] + mmRLC_SPARE_INT)*4;
745 grbm_cntl = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_CNTL_BASE_IDX] + mmGRBM_GFX_CNTL;
746 grbm_idx = adev->reg_offset[GC_HWIP][0][mmGRBM_GFX_INDEX_BASE_IDX] + mmGRBM_GFX_INDEX;
748 if (amdgpu_sriov_runtime(adev)) {
749 pr_err("shouldn't call rlcg write register during runtime\n");
753 if (offset == grbm_cntl || offset == grbm_idx) {
754 if (offset == grbm_cntl)
755 writel(v, scratch_reg2);
756 else if (offset == grbm_idx)
757 writel(v, scratch_reg3);
759 writel(v, ((void __iomem *)adev->rmmio) + (offset * 4));
762 uint32_t retries = 50000;
764 writel(v, scratch_reg0);
765 writel(offset | 0x80000000, scratch_reg1);
766 writel(1, spare_int);
767 for (i = 0; i < retries; i++) {
770 tmp = readl(scratch_reg1);
771 if (!(tmp & 0x80000000))
777 pr_err("timeout: rlcg program reg:0x%05x failed !\n", offset);
782 #define VEGA10_GB_ADDR_CONFIG_GOLDEN 0x2a114042
783 #define VEGA12_GB_ADDR_CONFIG_GOLDEN 0x24104041
784 #define RAVEN_GB_ADDR_CONFIG_GOLDEN 0x24000042
785 #define RAVEN2_GB_ADDR_CONFIG_GOLDEN 0x26013041
787 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev);
788 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev);
789 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev);
790 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev);
791 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
792 struct amdgpu_cu_info *cu_info);
793 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev);
794 static void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num, u32 instance);
795 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring);
796 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring);
797 static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
798 void *ras_error_status);
799 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev,
801 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev);
803 static void gfx_v9_0_kiq_set_resources(struct amdgpu_ring *kiq_ring,
806 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_SET_RESOURCES, 6));
807 amdgpu_ring_write(kiq_ring,
808 PACKET3_SET_RESOURCES_VMID_MASK(0) |
809 /* vmid_mask:0* queue_type:0 (KIQ) */
810 PACKET3_SET_RESOURCES_QUEUE_TYPE(0));
811 amdgpu_ring_write(kiq_ring,
812 lower_32_bits(queue_mask)); /* queue mask lo */
813 amdgpu_ring_write(kiq_ring,
814 upper_32_bits(queue_mask)); /* queue mask hi */
815 amdgpu_ring_write(kiq_ring, 0); /* gws mask lo */
816 amdgpu_ring_write(kiq_ring, 0); /* gws mask hi */
817 amdgpu_ring_write(kiq_ring, 0); /* oac mask */
818 amdgpu_ring_write(kiq_ring, 0); /* gds heap base:0, gds heap size:0 */
821 static void gfx_v9_0_kiq_map_queues(struct amdgpu_ring *kiq_ring,
822 struct amdgpu_ring *ring)
824 struct amdgpu_device *adev = kiq_ring->adev;
825 uint64_t mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj);
826 uint64_t wptr_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
827 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
829 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_MAP_QUEUES, 5));
830 /* Q_sel:0, vmid:0, vidmem: 1, engine:0, num_Q:1*/
831 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
832 PACKET3_MAP_QUEUES_QUEUE_SEL(0) | /* Queue_Sel */
833 PACKET3_MAP_QUEUES_VMID(0) | /* VMID */
834 PACKET3_MAP_QUEUES_QUEUE(ring->queue) |
835 PACKET3_MAP_QUEUES_PIPE(ring->pipe) |
836 PACKET3_MAP_QUEUES_ME((ring->me == 1 ? 0 : 1)) |
837 /*queue_type: normal compute queue */
838 PACKET3_MAP_QUEUES_QUEUE_TYPE(0) |
839 /* alloc format: all_on_one_pipe */
840 PACKET3_MAP_QUEUES_ALLOC_FORMAT(0) |
841 PACKET3_MAP_QUEUES_ENGINE_SEL(eng_sel) |
842 /* num_queues: must be 1 */
843 PACKET3_MAP_QUEUES_NUM_QUEUES(1));
844 amdgpu_ring_write(kiq_ring,
845 PACKET3_MAP_QUEUES_DOORBELL_OFFSET(ring->doorbell_index));
846 amdgpu_ring_write(kiq_ring, lower_32_bits(mqd_addr));
847 amdgpu_ring_write(kiq_ring, upper_32_bits(mqd_addr));
848 amdgpu_ring_write(kiq_ring, lower_32_bits(wptr_addr));
849 amdgpu_ring_write(kiq_ring, upper_32_bits(wptr_addr));
852 static void gfx_v9_0_kiq_unmap_queues(struct amdgpu_ring *kiq_ring,
853 struct amdgpu_ring *ring,
854 enum amdgpu_unmap_queues_action action,
855 u64 gpu_addr, u64 seq)
857 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
859 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_UNMAP_QUEUES, 4));
860 amdgpu_ring_write(kiq_ring, /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
861 PACKET3_UNMAP_QUEUES_ACTION(action) |
862 PACKET3_UNMAP_QUEUES_QUEUE_SEL(0) |
863 PACKET3_UNMAP_QUEUES_ENGINE_SEL(eng_sel) |
864 PACKET3_UNMAP_QUEUES_NUM_QUEUES(1));
865 amdgpu_ring_write(kiq_ring,
866 PACKET3_UNMAP_QUEUES_DOORBELL_OFFSET0(ring->doorbell_index));
868 if (action == PREEMPT_QUEUES_NO_UNMAP) {
869 amdgpu_ring_write(kiq_ring, lower_32_bits(gpu_addr));
870 amdgpu_ring_write(kiq_ring, upper_32_bits(gpu_addr));
871 amdgpu_ring_write(kiq_ring, seq);
873 amdgpu_ring_write(kiq_ring, 0);
874 amdgpu_ring_write(kiq_ring, 0);
875 amdgpu_ring_write(kiq_ring, 0);
879 static void gfx_v9_0_kiq_query_status(struct amdgpu_ring *kiq_ring,
880 struct amdgpu_ring *ring,
884 uint32_t eng_sel = ring->funcs->type == AMDGPU_RING_TYPE_GFX ? 4 : 0;
886 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_QUERY_STATUS, 5));
887 amdgpu_ring_write(kiq_ring,
888 PACKET3_QUERY_STATUS_CONTEXT_ID(0) |
889 PACKET3_QUERY_STATUS_INTERRUPT_SEL(0) |
890 PACKET3_QUERY_STATUS_COMMAND(2));
891 /* Q_sel: 0, vmid: 0, engine: 0, num_Q: 1 */
892 amdgpu_ring_write(kiq_ring,
893 PACKET3_QUERY_STATUS_DOORBELL_OFFSET(ring->doorbell_index) |
894 PACKET3_QUERY_STATUS_ENG_SEL(eng_sel));
895 amdgpu_ring_write(kiq_ring, lower_32_bits(addr));
896 amdgpu_ring_write(kiq_ring, upper_32_bits(addr));
897 amdgpu_ring_write(kiq_ring, lower_32_bits(seq));
898 amdgpu_ring_write(kiq_ring, upper_32_bits(seq));
901 static void gfx_v9_0_kiq_invalidate_tlbs(struct amdgpu_ring *kiq_ring,
902 uint16_t pasid, uint32_t flush_type,
905 amdgpu_ring_write(kiq_ring, PACKET3(PACKET3_INVALIDATE_TLBS, 0));
906 amdgpu_ring_write(kiq_ring,
907 PACKET3_INVALIDATE_TLBS_DST_SEL(1) |
908 PACKET3_INVALIDATE_TLBS_ALL_HUB(all_hub) |
909 PACKET3_INVALIDATE_TLBS_PASID(pasid) |
910 PACKET3_INVALIDATE_TLBS_FLUSH_TYPE(flush_type));
913 static const struct kiq_pm4_funcs gfx_v9_0_kiq_pm4_funcs = {
914 .kiq_set_resources = gfx_v9_0_kiq_set_resources,
915 .kiq_map_queues = gfx_v9_0_kiq_map_queues,
916 .kiq_unmap_queues = gfx_v9_0_kiq_unmap_queues,
917 .kiq_query_status = gfx_v9_0_kiq_query_status,
918 .kiq_invalidate_tlbs = gfx_v9_0_kiq_invalidate_tlbs,
919 .set_resources_size = 8,
920 .map_queues_size = 7,
921 .unmap_queues_size = 6,
922 .query_status_size = 7,
923 .invalidate_tlbs_size = 2,
926 static void gfx_v9_0_set_kiq_pm4_funcs(struct amdgpu_device *adev)
928 adev->gfx.kiq.pmf = &gfx_v9_0_kiq_pm4_funcs;
931 static void gfx_v9_0_init_golden_registers(struct amdgpu_device *adev)
933 switch (adev->asic_type) {
935 soc15_program_register_sequence(adev,
936 golden_settings_gc_9_0,
937 ARRAY_SIZE(golden_settings_gc_9_0));
938 soc15_program_register_sequence(adev,
939 golden_settings_gc_9_0_vg10,
940 ARRAY_SIZE(golden_settings_gc_9_0_vg10));
943 soc15_program_register_sequence(adev,
944 golden_settings_gc_9_2_1,
945 ARRAY_SIZE(golden_settings_gc_9_2_1));
946 soc15_program_register_sequence(adev,
947 golden_settings_gc_9_2_1_vg12,
948 ARRAY_SIZE(golden_settings_gc_9_2_1_vg12));
951 soc15_program_register_sequence(adev,
952 golden_settings_gc_9_0,
953 ARRAY_SIZE(golden_settings_gc_9_0));
954 soc15_program_register_sequence(adev,
955 golden_settings_gc_9_0_vg20,
956 ARRAY_SIZE(golden_settings_gc_9_0_vg20));
959 soc15_program_register_sequence(adev,
960 golden_settings_gc_9_4_1_arct,
961 ARRAY_SIZE(golden_settings_gc_9_4_1_arct));
964 soc15_program_register_sequence(adev, golden_settings_gc_9_1,
965 ARRAY_SIZE(golden_settings_gc_9_1));
966 if (adev->rev_id >= 8)
967 soc15_program_register_sequence(adev,
968 golden_settings_gc_9_1_rv2,
969 ARRAY_SIZE(golden_settings_gc_9_1_rv2));
971 soc15_program_register_sequence(adev,
972 golden_settings_gc_9_1_rv1,
973 ARRAY_SIZE(golden_settings_gc_9_1_rv1));
976 soc15_program_register_sequence(adev,
977 golden_settings_gc_9_1_rn,
978 ARRAY_SIZE(golden_settings_gc_9_1_rn));
979 return; /* for renoir, don't need common goldensetting */
984 if (adev->asic_type != CHIP_ARCTURUS)
985 soc15_program_register_sequence(adev, golden_settings_gc_9_x_common,
986 (const u32)ARRAY_SIZE(golden_settings_gc_9_x_common));
989 static void gfx_v9_0_scratch_init(struct amdgpu_device *adev)
991 adev->gfx.scratch.num_reg = 8;
992 adev->gfx.scratch.reg_base = SOC15_REG_OFFSET(GC, 0, mmSCRATCH_REG0);
993 adev->gfx.scratch.free_mask = (1u << adev->gfx.scratch.num_reg) - 1;
996 static void gfx_v9_0_write_data_to_reg(struct amdgpu_ring *ring, int eng_sel,
997 bool wc, uint32_t reg, uint32_t val)
999 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
1000 amdgpu_ring_write(ring, WRITE_DATA_ENGINE_SEL(eng_sel) |
1001 WRITE_DATA_DST_SEL(0) |
1002 (wc ? WR_CONFIRM : 0));
1003 amdgpu_ring_write(ring, reg);
1004 amdgpu_ring_write(ring, 0);
1005 amdgpu_ring_write(ring, val);
1008 static void gfx_v9_0_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
1009 int mem_space, int opt, uint32_t addr0,
1010 uint32_t addr1, uint32_t ref, uint32_t mask,
1013 amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
1014 amdgpu_ring_write(ring,
1015 /* memory (1) or register (0) */
1016 (WAIT_REG_MEM_MEM_SPACE(mem_space) |
1017 WAIT_REG_MEM_OPERATION(opt) | /* wait */
1018 WAIT_REG_MEM_FUNCTION(3) | /* equal */
1019 WAIT_REG_MEM_ENGINE(eng_sel)));
1022 BUG_ON(addr0 & 0x3); /* Dword align */
1023 amdgpu_ring_write(ring, addr0);
1024 amdgpu_ring_write(ring, addr1);
1025 amdgpu_ring_write(ring, ref);
1026 amdgpu_ring_write(ring, mask);
1027 amdgpu_ring_write(ring, inv); /* poll interval */
1030 static int gfx_v9_0_ring_test_ring(struct amdgpu_ring *ring)
1032 struct amdgpu_device *adev = ring->adev;
1038 r = amdgpu_gfx_scratch_get(adev, &scratch);
1042 WREG32(scratch, 0xCAFEDEAD);
1043 r = amdgpu_ring_alloc(ring, 3);
1045 goto error_free_scratch;
1047 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
1048 amdgpu_ring_write(ring, (scratch - PACKET3_SET_UCONFIG_REG_START));
1049 amdgpu_ring_write(ring, 0xDEADBEEF);
1050 amdgpu_ring_commit(ring);
1052 for (i = 0; i < adev->usec_timeout; i++) {
1053 tmp = RREG32(scratch);
1054 if (tmp == 0xDEADBEEF)
1059 if (i >= adev->usec_timeout)
1063 amdgpu_gfx_scratch_free(adev, scratch);
1067 static int gfx_v9_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
1069 struct amdgpu_device *adev = ring->adev;
1070 struct amdgpu_ib ib;
1071 struct dma_fence *f = NULL;
1078 r = amdgpu_device_wb_get(adev, &index);
1082 gpu_addr = adev->wb.gpu_addr + (index * 4);
1083 adev->wb.wb[index] = cpu_to_le32(0xCAFEDEAD);
1084 memset(&ib, 0, sizeof(ib));
1085 r = amdgpu_ib_get(adev, NULL, 16, &ib);
1089 ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
1090 ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
1091 ib.ptr[2] = lower_32_bits(gpu_addr);
1092 ib.ptr[3] = upper_32_bits(gpu_addr);
1093 ib.ptr[4] = 0xDEADBEEF;
1096 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
1100 r = dma_fence_wait_timeout(f, false, timeout);
1108 tmp = adev->wb.wb[index];
1109 if (tmp == 0xDEADBEEF)
1115 amdgpu_ib_free(adev, &ib, NULL);
1118 amdgpu_device_wb_free(adev, index);
1123 static void gfx_v9_0_free_microcode(struct amdgpu_device *adev)
1125 release_firmware(adev->gfx.pfp_fw);
1126 adev->gfx.pfp_fw = NULL;
1127 release_firmware(adev->gfx.me_fw);
1128 adev->gfx.me_fw = NULL;
1129 release_firmware(adev->gfx.ce_fw);
1130 adev->gfx.ce_fw = NULL;
1131 release_firmware(adev->gfx.rlc_fw);
1132 adev->gfx.rlc_fw = NULL;
1133 release_firmware(adev->gfx.mec_fw);
1134 adev->gfx.mec_fw = NULL;
1135 release_firmware(adev->gfx.mec2_fw);
1136 adev->gfx.mec2_fw = NULL;
1138 kfree(adev->gfx.rlc.register_list_format);
1141 static void gfx_v9_0_init_rlc_ext_microcode(struct amdgpu_device *adev)
1143 const struct rlc_firmware_header_v2_1 *rlc_hdr;
1145 rlc_hdr = (const struct rlc_firmware_header_v2_1 *)adev->gfx.rlc_fw->data;
1146 adev->gfx.rlc_srlc_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_ucode_ver);
1147 adev->gfx.rlc_srlc_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_cntl_feature_ver);
1148 adev->gfx.rlc.save_restore_list_cntl_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_cntl_size_bytes);
1149 adev->gfx.rlc.save_restore_list_cntl = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_cntl_offset_bytes);
1150 adev->gfx.rlc_srlg_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_ucode_ver);
1151 adev->gfx.rlc_srlg_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_gpm_feature_ver);
1152 adev->gfx.rlc.save_restore_list_gpm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_gpm_size_bytes);
1153 adev->gfx.rlc.save_restore_list_gpm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_gpm_offset_bytes);
1154 adev->gfx.rlc_srls_fw_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_ucode_ver);
1155 adev->gfx.rlc_srls_feature_version = le32_to_cpu(rlc_hdr->save_restore_list_srm_feature_ver);
1156 adev->gfx.rlc.save_restore_list_srm_size_bytes = le32_to_cpu(rlc_hdr->save_restore_list_srm_size_bytes);
1157 adev->gfx.rlc.save_restore_list_srm = (u8 *)rlc_hdr + le32_to_cpu(rlc_hdr->save_restore_list_srm_offset_bytes);
1158 adev->gfx.rlc.reg_list_format_direct_reg_list_length =
1159 le32_to_cpu(rlc_hdr->reg_list_format_direct_reg_list_length);
1162 static void gfx_v9_0_check_fw_write_wait(struct amdgpu_device *adev)
1164 adev->gfx.me_fw_write_wait = false;
1165 adev->gfx.mec_fw_write_wait = false;
1167 if ((adev->asic_type != CHIP_ARCTURUS) &&
1168 ((adev->gfx.mec_fw_version < 0x000001a5) ||
1169 (adev->gfx.mec_feature_version < 46) ||
1170 (adev->gfx.pfp_fw_version < 0x000000b7) ||
1171 (adev->gfx.pfp_feature_version < 46)))
1172 DRM_WARN_ONCE("CP firmware version too old, please update!");
1174 switch (adev->asic_type) {
1176 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1177 (adev->gfx.me_feature_version >= 42) &&
1178 (adev->gfx.pfp_fw_version >= 0x000000b1) &&
1179 (adev->gfx.pfp_feature_version >= 42))
1180 adev->gfx.me_fw_write_wait = true;
1182 if ((adev->gfx.mec_fw_version >= 0x00000193) &&
1183 (adev->gfx.mec_feature_version >= 42))
1184 adev->gfx.mec_fw_write_wait = true;
1187 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1188 (adev->gfx.me_feature_version >= 44) &&
1189 (adev->gfx.pfp_fw_version >= 0x000000b2) &&
1190 (adev->gfx.pfp_feature_version >= 44))
1191 adev->gfx.me_fw_write_wait = true;
1193 if ((adev->gfx.mec_fw_version >= 0x00000196) &&
1194 (adev->gfx.mec_feature_version >= 44))
1195 adev->gfx.mec_fw_write_wait = true;
1198 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1199 (adev->gfx.me_feature_version >= 44) &&
1200 (adev->gfx.pfp_fw_version >= 0x000000b2) &&
1201 (adev->gfx.pfp_feature_version >= 44))
1202 adev->gfx.me_fw_write_wait = true;
1204 if ((adev->gfx.mec_fw_version >= 0x00000197) &&
1205 (adev->gfx.mec_feature_version >= 44))
1206 adev->gfx.mec_fw_write_wait = true;
1209 if ((adev->gfx.me_fw_version >= 0x0000009c) &&
1210 (adev->gfx.me_feature_version >= 42) &&
1211 (adev->gfx.pfp_fw_version >= 0x000000b1) &&
1212 (adev->gfx.pfp_feature_version >= 42))
1213 adev->gfx.me_fw_write_wait = true;
1215 if ((adev->gfx.mec_fw_version >= 0x00000192) &&
1216 (adev->gfx.mec_feature_version >= 42))
1217 adev->gfx.mec_fw_write_wait = true;
1220 adev->gfx.me_fw_write_wait = true;
1221 adev->gfx.mec_fw_write_wait = true;
1226 struct amdgpu_gfxoff_quirk {
1234 static const struct amdgpu_gfxoff_quirk amdgpu_gfxoff_quirk_list[] = {
1235 /* https://bugzilla.kernel.org/show_bug.cgi?id=204689 */
1236 { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc8 },
1237 /* https://bugzilla.kernel.org/show_bug.cgi?id=207171 */
1238 { 0x1002, 0x15dd, 0x103c, 0x83e7, 0xd3 },
1242 static bool gfx_v9_0_should_disable_gfxoff(struct pci_dev *pdev)
1244 const struct amdgpu_gfxoff_quirk *p = amdgpu_gfxoff_quirk_list;
1246 while (p && p->chip_device != 0) {
1247 if (pdev->vendor == p->chip_vendor &&
1248 pdev->device == p->chip_device &&
1249 pdev->subsystem_vendor == p->subsys_vendor &&
1250 pdev->subsystem_device == p->subsys_device &&
1251 pdev->revision == p->revision) {
1259 static bool is_raven_kicker(struct amdgpu_device *adev)
1261 if (adev->pm.fw_version >= 0x41e2b)
1267 static void gfx_v9_0_check_if_need_gfxoff(struct amdgpu_device *adev)
1269 if (gfx_v9_0_should_disable_gfxoff(adev->pdev))
1270 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
1272 switch (adev->asic_type) {
1278 if (!(adev->rev_id >= 0x8 || adev->pdev->device == 0x15d8) &&
1279 ((!is_raven_kicker(adev) &&
1280 adev->gfx.rlc_fw_version < 531) ||
1281 (adev->gfx.rlc_feature_version < 1) ||
1282 !adev->gfx.rlc.is_rlc_v2_1))
1283 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
1285 if (adev->pm.pp_feature & PP_GFXOFF_MASK)
1286 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
1288 AMD_PG_SUPPORT_RLC_SMU_HS;
1291 if (adev->pm.pp_feature & PP_GFXOFF_MASK)
1292 adev->pg_flags |= AMD_PG_SUPPORT_GFX_PG |
1294 AMD_PG_SUPPORT_RLC_SMU_HS;
1301 static int gfx_v9_0_init_cp_gfx_microcode(struct amdgpu_device *adev,
1302 const char *chip_name)
1306 struct amdgpu_firmware_info *info = NULL;
1307 const struct common_firmware_header *header = NULL;
1308 const struct gfx_firmware_header_v1_0 *cp_hdr;
1310 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_pfp.bin", chip_name);
1311 err = request_firmware(&adev->gfx.pfp_fw, fw_name, adev->dev);
1314 err = amdgpu_ucode_validate(adev->gfx.pfp_fw);
1317 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data;
1318 adev->gfx.pfp_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1319 adev->gfx.pfp_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1321 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_me.bin", chip_name);
1322 err = request_firmware(&adev->gfx.me_fw, fw_name, adev->dev);
1325 err = amdgpu_ucode_validate(adev->gfx.me_fw);
1328 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data;
1329 adev->gfx.me_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1330 adev->gfx.me_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1332 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ce.bin", chip_name);
1333 err = request_firmware(&adev->gfx.ce_fw, fw_name, adev->dev);
1336 err = amdgpu_ucode_validate(adev->gfx.ce_fw);
1339 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data;
1340 adev->gfx.ce_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1341 adev->gfx.ce_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1343 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1344 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_PFP];
1345 info->ucode_id = AMDGPU_UCODE_ID_CP_PFP;
1346 info->fw = adev->gfx.pfp_fw;
1347 header = (const struct common_firmware_header *)info->fw->data;
1348 adev->firmware.fw_size +=
1349 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1351 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_ME];
1352 info->ucode_id = AMDGPU_UCODE_ID_CP_ME;
1353 info->fw = adev->gfx.me_fw;
1354 header = (const struct common_firmware_header *)info->fw->data;
1355 adev->firmware.fw_size +=
1356 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1358 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_CE];
1359 info->ucode_id = AMDGPU_UCODE_ID_CP_CE;
1360 info->fw = adev->gfx.ce_fw;
1361 header = (const struct common_firmware_header *)info->fw->data;
1362 adev->firmware.fw_size +=
1363 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1369 "gfx9: Failed to load firmware \"%s\"\n",
1371 release_firmware(adev->gfx.pfp_fw);
1372 adev->gfx.pfp_fw = NULL;
1373 release_firmware(adev->gfx.me_fw);
1374 adev->gfx.me_fw = NULL;
1375 release_firmware(adev->gfx.ce_fw);
1376 adev->gfx.ce_fw = NULL;
1381 static int gfx_v9_0_init_rlc_microcode(struct amdgpu_device *adev,
1382 const char *chip_name)
1386 struct amdgpu_firmware_info *info = NULL;
1387 const struct common_firmware_header *header = NULL;
1388 const struct rlc_firmware_header_v2_0 *rlc_hdr;
1389 unsigned int *tmp = NULL;
1391 uint16_t version_major;
1392 uint16_t version_minor;
1393 uint32_t smu_version;
1396 * For Picasso && AM4 SOCKET board, we use picasso_rlc_am4.bin
1397 * instead of picasso_rlc.bin.
1399 * PCO AM4: revision >= 0xC8 && revision <= 0xCF
1400 * or revision >= 0xD8 && revision <= 0xDF
1401 * otherwise is PCO FP5
1403 if (!strcmp(chip_name, "picasso") &&
1404 (((adev->pdev->revision >= 0xC8) && (adev->pdev->revision <= 0xCF)) ||
1405 ((adev->pdev->revision >= 0xD8) && (adev->pdev->revision <= 0xDF))))
1406 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc_am4.bin", chip_name);
1407 else if (!strcmp(chip_name, "raven") && (amdgpu_pm_load_smu_firmware(adev, &smu_version) == 0) &&
1408 (smu_version >= 0x41e2b))
1410 *SMC is loaded by SBIOS on APU and it's able to get the SMU version directly.
1412 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_kicker_rlc.bin", chip_name);
1414 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_rlc.bin", chip_name);
1415 err = request_firmware(&adev->gfx.rlc_fw, fw_name, adev->dev);
1418 err = amdgpu_ucode_validate(adev->gfx.rlc_fw);
1419 rlc_hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
1421 version_major = le16_to_cpu(rlc_hdr->header.header_version_major);
1422 version_minor = le16_to_cpu(rlc_hdr->header.header_version_minor);
1423 if (version_major == 2 && version_minor == 1)
1424 adev->gfx.rlc.is_rlc_v2_1 = true;
1426 adev->gfx.rlc_fw_version = le32_to_cpu(rlc_hdr->header.ucode_version);
1427 adev->gfx.rlc_feature_version = le32_to_cpu(rlc_hdr->ucode_feature_version);
1428 adev->gfx.rlc.save_and_restore_offset =
1429 le32_to_cpu(rlc_hdr->save_and_restore_offset);
1430 adev->gfx.rlc.clear_state_descriptor_offset =
1431 le32_to_cpu(rlc_hdr->clear_state_descriptor_offset);
1432 adev->gfx.rlc.avail_scratch_ram_locations =
1433 le32_to_cpu(rlc_hdr->avail_scratch_ram_locations);
1434 adev->gfx.rlc.reg_restore_list_size =
1435 le32_to_cpu(rlc_hdr->reg_restore_list_size);
1436 adev->gfx.rlc.reg_list_format_start =
1437 le32_to_cpu(rlc_hdr->reg_list_format_start);
1438 adev->gfx.rlc.reg_list_format_separate_start =
1439 le32_to_cpu(rlc_hdr->reg_list_format_separate_start);
1440 adev->gfx.rlc.starting_offsets_start =
1441 le32_to_cpu(rlc_hdr->starting_offsets_start);
1442 adev->gfx.rlc.reg_list_format_size_bytes =
1443 le32_to_cpu(rlc_hdr->reg_list_format_size_bytes);
1444 adev->gfx.rlc.reg_list_size_bytes =
1445 le32_to_cpu(rlc_hdr->reg_list_size_bytes);
1446 adev->gfx.rlc.register_list_format =
1447 kmalloc(adev->gfx.rlc.reg_list_format_size_bytes +
1448 adev->gfx.rlc.reg_list_size_bytes, GFP_KERNEL);
1449 if (!adev->gfx.rlc.register_list_format) {
1454 tmp = (unsigned int *)((uintptr_t)rlc_hdr +
1455 le32_to_cpu(rlc_hdr->reg_list_format_array_offset_bytes));
1456 for (i = 0 ; i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2); i++)
1457 adev->gfx.rlc.register_list_format[i] = le32_to_cpu(tmp[i]);
1459 adev->gfx.rlc.register_restore = adev->gfx.rlc.register_list_format + i;
1461 tmp = (unsigned int *)((uintptr_t)rlc_hdr +
1462 le32_to_cpu(rlc_hdr->reg_list_array_offset_bytes));
1463 for (i = 0 ; i < (adev->gfx.rlc.reg_list_size_bytes >> 2); i++)
1464 adev->gfx.rlc.register_restore[i] = le32_to_cpu(tmp[i]);
1466 if (adev->gfx.rlc.is_rlc_v2_1)
1467 gfx_v9_0_init_rlc_ext_microcode(adev);
1469 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1470 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_G];
1471 info->ucode_id = AMDGPU_UCODE_ID_RLC_G;
1472 info->fw = adev->gfx.rlc_fw;
1473 header = (const struct common_firmware_header *)info->fw->data;
1474 adev->firmware.fw_size +=
1475 ALIGN(le32_to_cpu(header->ucode_size_bytes), PAGE_SIZE);
1477 if (adev->gfx.rlc.is_rlc_v2_1 &&
1478 adev->gfx.rlc.save_restore_list_cntl_size_bytes &&
1479 adev->gfx.rlc.save_restore_list_gpm_size_bytes &&
1480 adev->gfx.rlc.save_restore_list_srm_size_bytes) {
1481 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL];
1482 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL;
1483 info->fw = adev->gfx.rlc_fw;
1484 adev->firmware.fw_size +=
1485 ALIGN(adev->gfx.rlc.save_restore_list_cntl_size_bytes, PAGE_SIZE);
1487 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM];
1488 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM;
1489 info->fw = adev->gfx.rlc_fw;
1490 adev->firmware.fw_size +=
1491 ALIGN(adev->gfx.rlc.save_restore_list_gpm_size_bytes, PAGE_SIZE);
1493 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM];
1494 info->ucode_id = AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM;
1495 info->fw = adev->gfx.rlc_fw;
1496 adev->firmware.fw_size +=
1497 ALIGN(adev->gfx.rlc.save_restore_list_srm_size_bytes, PAGE_SIZE);
1504 "gfx9: Failed to load firmware \"%s\"\n",
1506 release_firmware(adev->gfx.rlc_fw);
1507 adev->gfx.rlc_fw = NULL;
1512 static int gfx_v9_0_init_cp_compute_microcode(struct amdgpu_device *adev,
1513 const char *chip_name)
1517 struct amdgpu_firmware_info *info = NULL;
1518 const struct common_firmware_header *header = NULL;
1519 const struct gfx_firmware_header_v1_0 *cp_hdr;
1521 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec.bin", chip_name);
1522 err = request_firmware(&adev->gfx.mec_fw, fw_name, adev->dev);
1525 err = amdgpu_ucode_validate(adev->gfx.mec_fw);
1528 cp_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1529 adev->gfx.mec_fw_version = le32_to_cpu(cp_hdr->header.ucode_version);
1530 adev->gfx.mec_feature_version = le32_to_cpu(cp_hdr->ucode_feature_version);
1533 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mec2.bin", chip_name);
1534 err = request_firmware(&adev->gfx.mec2_fw, fw_name, adev->dev);
1536 err = amdgpu_ucode_validate(adev->gfx.mec2_fw);
1539 cp_hdr = (const struct gfx_firmware_header_v1_0 *)
1540 adev->gfx.mec2_fw->data;
1541 adev->gfx.mec2_fw_version =
1542 le32_to_cpu(cp_hdr->header.ucode_version);
1543 adev->gfx.mec2_feature_version =
1544 le32_to_cpu(cp_hdr->ucode_feature_version);
1547 adev->gfx.mec2_fw = NULL;
1550 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
1551 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1];
1552 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1;
1553 info->fw = adev->gfx.mec_fw;
1554 header = (const struct common_firmware_header *)info->fw->data;
1555 cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data;
1556 adev->firmware.fw_size +=
1557 ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
1559 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC1_JT];
1560 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC1_JT;
1561 info->fw = adev->gfx.mec_fw;
1562 adev->firmware.fw_size +=
1563 ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
1565 if (adev->gfx.mec2_fw) {
1566 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2];
1567 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
1568 info->fw = adev->gfx.mec2_fw;
1569 header = (const struct common_firmware_header *)info->fw->data;
1570 cp_hdr = (const struct gfx_firmware_header_v1_0 *)info->fw->data;
1571 adev->firmware.fw_size +=
1572 ALIGN(le32_to_cpu(header->ucode_size_bytes) - le32_to_cpu(cp_hdr->jt_size) * 4, PAGE_SIZE);
1574 /* TODO: Determine if MEC2 JT FW loading can be removed
1575 for all GFX V9 asic and above */
1576 if (adev->asic_type != CHIP_ARCTURUS &&
1577 adev->asic_type != CHIP_RENOIR) {
1578 info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CP_MEC2_JT];
1579 info->ucode_id = AMDGPU_UCODE_ID_CP_MEC2_JT;
1580 info->fw = adev->gfx.mec2_fw;
1581 adev->firmware.fw_size +=
1582 ALIGN(le32_to_cpu(cp_hdr->jt_size) * 4,
1589 gfx_v9_0_check_if_need_gfxoff(adev);
1590 gfx_v9_0_check_fw_write_wait(adev);
1593 "gfx9: Failed to load firmware \"%s\"\n",
1595 release_firmware(adev->gfx.mec_fw);
1596 adev->gfx.mec_fw = NULL;
1597 release_firmware(adev->gfx.mec2_fw);
1598 adev->gfx.mec2_fw = NULL;
1603 static int gfx_v9_0_init_microcode(struct amdgpu_device *adev)
1605 const char *chip_name;
1610 switch (adev->asic_type) {
1612 chip_name = "vega10";
1615 chip_name = "vega12";
1618 chip_name = "vega20";
1621 if (adev->rev_id >= 8)
1622 chip_name = "raven2";
1623 else if (adev->pdev->device == 0x15d8)
1624 chip_name = "picasso";
1626 chip_name = "raven";
1629 chip_name = "arcturus";
1632 chip_name = "renoir";
1638 /* No CPG in Arcturus */
1639 if (adev->asic_type != CHIP_ARCTURUS) {
1640 r = gfx_v9_0_init_cp_gfx_microcode(adev, chip_name);
1645 r = gfx_v9_0_init_rlc_microcode(adev, chip_name);
1649 r = gfx_v9_0_init_cp_compute_microcode(adev, chip_name);
1656 static u32 gfx_v9_0_get_csb_size(struct amdgpu_device *adev)
1659 const struct cs_section_def *sect = NULL;
1660 const struct cs_extent_def *ext = NULL;
1662 /* begin clear state */
1664 /* context control state */
1667 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {
1668 for (ext = sect->section; ext->extent != NULL; ++ext) {
1669 if (sect->id == SECT_CONTEXT)
1670 count += 2 + ext->reg_count;
1676 /* end clear state */
1684 static void gfx_v9_0_get_csb_buffer(struct amdgpu_device *adev,
1685 volatile u32 *buffer)
1688 const struct cs_section_def *sect = NULL;
1689 const struct cs_extent_def *ext = NULL;
1691 if (adev->gfx.rlc.cs_data == NULL)
1696 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1697 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1699 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
1700 buffer[count++] = cpu_to_le32(0x80000000);
1701 buffer[count++] = cpu_to_le32(0x80000000);
1703 for (sect = adev->gfx.rlc.cs_data; sect->section != NULL; ++sect) {
1704 for (ext = sect->section; ext->extent != NULL; ++ext) {
1705 if (sect->id == SECT_CONTEXT) {
1707 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
1708 buffer[count++] = cpu_to_le32(ext->reg_index -
1709 PACKET3_SET_CONTEXT_REG_START);
1710 for (i = 0; i < ext->reg_count; i++)
1711 buffer[count++] = cpu_to_le32(ext->extent[i]);
1718 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1719 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
1721 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
1722 buffer[count++] = cpu_to_le32(0);
1725 static void gfx_v9_0_init_always_on_cu_mask(struct amdgpu_device *adev)
1727 struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info;
1728 uint32_t pg_always_on_cu_num = 2;
1729 uint32_t always_on_cu_num;
1731 uint32_t mask, cu_bitmap, counter;
1733 if (adev->flags & AMD_IS_APU)
1734 always_on_cu_num = 4;
1735 else if (adev->asic_type == CHIP_VEGA12)
1736 always_on_cu_num = 8;
1738 always_on_cu_num = 12;
1740 mutex_lock(&adev->grbm_idx_mutex);
1741 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
1742 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
1746 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
1748 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
1749 if (cu_info->bitmap[i][j] & mask) {
1750 if (counter == pg_always_on_cu_num)
1751 WREG32_SOC15(GC, 0, mmRLC_PG_ALWAYS_ON_CU_MASK, cu_bitmap);
1752 if (counter < always_on_cu_num)
1761 WREG32_SOC15(GC, 0, mmRLC_LB_ALWAYS_ACTIVE_CU_MASK, cu_bitmap);
1762 cu_info->ao_cu_bitmap[i][j] = cu_bitmap;
1765 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1766 mutex_unlock(&adev->grbm_idx_mutex);
1769 static void gfx_v9_0_init_lbpw(struct amdgpu_device *adev)
1773 /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */
1774 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);
1775 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x0333A5A7);
1776 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);
1777 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x30 | 0x40 << 8 | 0x02FA << 16));
1779 /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */
1780 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);
1782 /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */
1783 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000500);
1785 mutex_lock(&adev->grbm_idx_mutex);
1786 /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/
1787 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1788 WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);
1790 /* set mmRLC_LB_PARAMS = 0x003F_1006 */
1791 data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);
1792 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);
1793 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);
1794 WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);
1796 /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */
1797 data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);
1800 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);
1803 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xF (4 CUs AON for Raven),
1804 * programmed in gfx_v9_0_init_always_on_cu_mask()
1807 /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved,
1808 * but used for RLC_LB_CNTL configuration */
1809 data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK;
1810 data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);
1811 data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);
1812 WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);
1813 mutex_unlock(&adev->grbm_idx_mutex);
1815 gfx_v9_0_init_always_on_cu_mask(adev);
1818 static void gfx_v9_4_init_lbpw(struct amdgpu_device *adev)
1822 /* set mmRLC_LB_THR_CONFIG_1/2/3/4 */
1823 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_1, 0x0000007F);
1824 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_2, 0x033388F8);
1825 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_3, 0x00000077);
1826 WREG32_SOC15(GC, 0, mmRLC_LB_THR_CONFIG_4, (0x10 | 0x27 << 8 | 0x02FA << 16));
1828 /* set mmRLC_LB_CNTR_INIT = 0x0000_0000 */
1829 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_INIT, 0x00000000);
1831 /* set mmRLC_LB_CNTR_MAX = 0x0000_0500 */
1832 WREG32_SOC15(GC, 0, mmRLC_LB_CNTR_MAX, 0x00000800);
1834 mutex_lock(&adev->grbm_idx_mutex);
1835 /* set mmRLC_LB_INIT_CU_MASK thru broadcast mode to enable all SE/SH*/
1836 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1837 WREG32_SOC15(GC, 0, mmRLC_LB_INIT_CU_MASK, 0xffffffff);
1839 /* set mmRLC_LB_PARAMS = 0x003F_1006 */
1840 data = REG_SET_FIELD(0, RLC_LB_PARAMS, FIFO_SAMPLES, 0x0003);
1841 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLES, 0x0010);
1842 data |= REG_SET_FIELD(data, RLC_LB_PARAMS, PG_IDLE_SAMPLE_INTERVAL, 0x033F);
1843 WREG32_SOC15(GC, 0, mmRLC_LB_PARAMS, data);
1845 /* set mmRLC_GPM_GENERAL_7[31-16] = 0x00C0 */
1846 data = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7);
1849 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_7, data);
1852 * RLC_LB_ALWAYS_ACTIVE_CU_MASK = 0xFFF (12 CUs AON),
1853 * programmed in gfx_v9_0_init_always_on_cu_mask()
1856 /* set RLC_LB_CNTL = 0x8000_0095, 31 bit is reserved,
1857 * but used for RLC_LB_CNTL configuration */
1858 data = RLC_LB_CNTL__LB_CNT_SPIM_ACTIVE_MASK;
1859 data |= REG_SET_FIELD(data, RLC_LB_CNTL, CU_MASK_USED_OFF_HYST, 0x09);
1860 data |= REG_SET_FIELD(data, RLC_LB_CNTL, RESERVED, 0x80000);
1861 WREG32_SOC15(GC, 0, mmRLC_LB_CNTL, data);
1862 mutex_unlock(&adev->grbm_idx_mutex);
1864 gfx_v9_0_init_always_on_cu_mask(adev);
1867 static void gfx_v9_0_enable_lbpw(struct amdgpu_device *adev, bool enable)
1869 WREG32_FIELD15(GC, 0, RLC_LB_CNTL, LOAD_BALANCE_ENABLE, enable ? 1 : 0);
1872 static int gfx_v9_0_cp_jump_table_num(struct amdgpu_device *adev)
1877 static int gfx_v9_0_rlc_init(struct amdgpu_device *adev)
1879 const struct cs_section_def *cs_data;
1882 adev->gfx.rlc.cs_data = gfx9_cs_data;
1884 cs_data = adev->gfx.rlc.cs_data;
1887 /* init clear state block */
1888 r = amdgpu_gfx_rlc_init_csb(adev);
1893 if (adev->asic_type == CHIP_RAVEN || adev->asic_type == CHIP_RENOIR) {
1894 /* TODO: double check the cp_table_size for RV */
1895 adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */
1896 r = amdgpu_gfx_rlc_init_cpt(adev);
1901 switch (adev->asic_type) {
1903 gfx_v9_0_init_lbpw(adev);
1906 gfx_v9_4_init_lbpw(adev);
1912 /* init spm vmid with 0xf */
1913 if (adev->gfx.rlc.funcs->update_spm_vmid)
1914 adev->gfx.rlc.funcs->update_spm_vmid(adev, 0xf);
1919 static void gfx_v9_0_mec_fini(struct amdgpu_device *adev)
1921 amdgpu_bo_free_kernel(&adev->gfx.mec.hpd_eop_obj, NULL, NULL);
1922 amdgpu_bo_free_kernel(&adev->gfx.mec.mec_fw_obj, NULL, NULL);
1925 static int gfx_v9_0_mec_init(struct amdgpu_device *adev)
1929 const __le32 *fw_data;
1932 size_t mec_hpd_size;
1934 const struct gfx_firmware_header_v1_0 *mec_hdr;
1936 bitmap_zero(adev->gfx.mec.queue_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
1938 /* take ownership of the relevant compute queues */
1939 amdgpu_gfx_compute_queue_acquire(adev);
1940 mec_hpd_size = adev->gfx.num_compute_rings * GFX9_MEC_HPD_SIZE;
1942 r = amdgpu_bo_create_reserved(adev, mec_hpd_size, PAGE_SIZE,
1943 AMDGPU_GEM_DOMAIN_VRAM,
1944 &adev->gfx.mec.hpd_eop_obj,
1945 &adev->gfx.mec.hpd_eop_gpu_addr,
1948 dev_warn(adev->dev, "(%d) create HDP EOP bo failed\n", r);
1949 gfx_v9_0_mec_fini(adev);
1953 memset(hpd, 0, mec_hpd_size);
1955 amdgpu_bo_kunmap(adev->gfx.mec.hpd_eop_obj);
1956 amdgpu_bo_unreserve(adev->gfx.mec.hpd_eop_obj);
1958 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
1960 fw_data = (const __le32 *)
1961 (adev->gfx.mec_fw->data +
1962 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
1963 fw_size = le32_to_cpu(mec_hdr->header.ucode_size_bytes) / 4;
1965 r = amdgpu_bo_create_reserved(adev, mec_hdr->header.ucode_size_bytes,
1966 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
1967 &adev->gfx.mec.mec_fw_obj,
1968 &adev->gfx.mec.mec_fw_gpu_addr,
1971 dev_warn(adev->dev, "(%d) create mec firmware bo failed\n", r);
1972 gfx_v9_0_mec_fini(adev);
1976 memcpy(fw, fw_data, fw_size);
1978 amdgpu_bo_kunmap(adev->gfx.mec.mec_fw_obj);
1979 amdgpu_bo_unreserve(adev->gfx.mec.mec_fw_obj);
1984 static uint32_t wave_read_ind(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t address)
1986 WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX,
1987 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
1988 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
1989 (address << SQ_IND_INDEX__INDEX__SHIFT) |
1990 (SQ_IND_INDEX__FORCE_READ_MASK));
1991 return RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
1994 static void wave_read_regs(struct amdgpu_device *adev, uint32_t simd,
1995 uint32_t wave, uint32_t thread,
1996 uint32_t regno, uint32_t num, uint32_t *out)
1998 WREG32_SOC15_RLC(GC, 0, mmSQ_IND_INDEX,
1999 (wave << SQ_IND_INDEX__WAVE_ID__SHIFT) |
2000 (simd << SQ_IND_INDEX__SIMD_ID__SHIFT) |
2001 (regno << SQ_IND_INDEX__INDEX__SHIFT) |
2002 (thread << SQ_IND_INDEX__THREAD_ID__SHIFT) |
2003 (SQ_IND_INDEX__FORCE_READ_MASK) |
2004 (SQ_IND_INDEX__AUTO_INCR_MASK));
2006 *(out++) = RREG32_SOC15(GC, 0, mmSQ_IND_DATA);
2009 static void gfx_v9_0_read_wave_data(struct amdgpu_device *adev, uint32_t simd, uint32_t wave, uint32_t *dst, int *no_fields)
2011 /* type 1 wave data */
2012 dst[(*no_fields)++] = 1;
2013 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_STATUS);
2014 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_LO);
2015 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_PC_HI);
2016 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_LO);
2017 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_EXEC_HI);
2018 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_HW_ID);
2019 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW0);
2020 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_INST_DW1);
2021 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_GPR_ALLOC);
2022 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_LDS_ALLOC);
2023 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_TRAPSTS);
2024 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_STS);
2025 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_IB_DBG0);
2026 dst[(*no_fields)++] = wave_read_ind(adev, simd, wave, ixSQ_WAVE_M0);
2029 static void gfx_v9_0_read_wave_sgprs(struct amdgpu_device *adev, uint32_t simd,
2030 uint32_t wave, uint32_t start,
2031 uint32_t size, uint32_t *dst)
2034 adev, simd, wave, 0,
2035 start + SQIND_WAVE_SGPRS_OFFSET, size, dst);
2038 static void gfx_v9_0_read_wave_vgprs(struct amdgpu_device *adev, uint32_t simd,
2039 uint32_t wave, uint32_t thread,
2040 uint32_t start, uint32_t size,
2044 adev, simd, wave, thread,
2045 start + SQIND_WAVE_VGPRS_OFFSET, size, dst);
2048 static void gfx_v9_0_select_me_pipe_q(struct amdgpu_device *adev,
2049 u32 me, u32 pipe, u32 q, u32 vm)
2051 soc15_grbm_select(adev, me, pipe, q, vm);
2054 static const struct amdgpu_gfx_funcs gfx_v9_0_gfx_funcs = {
2055 .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter,
2056 .select_se_sh = &gfx_v9_0_select_se_sh,
2057 .read_wave_data = &gfx_v9_0_read_wave_data,
2058 .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs,
2059 .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs,
2060 .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q,
2061 .ras_error_inject = &gfx_v9_0_ras_error_inject,
2062 .query_ras_error_count = &gfx_v9_0_query_ras_error_count,
2063 .reset_ras_error_count = &gfx_v9_0_reset_ras_error_count,
2066 static const struct amdgpu_gfx_funcs gfx_v9_4_gfx_funcs = {
2067 .get_gpu_clock_counter = &gfx_v9_0_get_gpu_clock_counter,
2068 .select_se_sh = &gfx_v9_0_select_se_sh,
2069 .read_wave_data = &gfx_v9_0_read_wave_data,
2070 .read_wave_sgprs = &gfx_v9_0_read_wave_sgprs,
2071 .read_wave_vgprs = &gfx_v9_0_read_wave_vgprs,
2072 .select_me_pipe_q = &gfx_v9_0_select_me_pipe_q,
2073 .ras_error_inject = &gfx_v9_4_ras_error_inject,
2074 .query_ras_error_count = &gfx_v9_4_query_ras_error_count,
2075 .reset_ras_error_count = &gfx_v9_4_reset_ras_error_count,
2078 static int gfx_v9_0_gpu_early_init(struct amdgpu_device *adev)
2083 adev->gfx.funcs = &gfx_v9_0_gfx_funcs;
2085 switch (adev->asic_type) {
2087 adev->gfx.config.max_hw_contexts = 8;
2088 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2089 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2090 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2091 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2092 gb_addr_config = VEGA10_GB_ADDR_CONFIG_GOLDEN;
2095 adev->gfx.config.max_hw_contexts = 8;
2096 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2097 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2098 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2099 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2100 gb_addr_config = VEGA12_GB_ADDR_CONFIG_GOLDEN;
2101 DRM_INFO("fix gfx.config for vega12\n");
2104 adev->gfx.config.max_hw_contexts = 8;
2105 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2106 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2107 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2108 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2109 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2110 gb_addr_config &= ~0xf3e777ff;
2111 gb_addr_config |= 0x22014042;
2112 /* check vbios table if gpu info is not available */
2113 err = amdgpu_atomfirmware_get_gfx_info(adev);
2118 adev->gfx.config.max_hw_contexts = 8;
2119 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2120 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2121 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2122 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2123 if (adev->rev_id >= 8)
2124 gb_addr_config = RAVEN2_GB_ADDR_CONFIG_GOLDEN;
2126 gb_addr_config = RAVEN_GB_ADDR_CONFIG_GOLDEN;
2129 adev->gfx.funcs = &gfx_v9_4_gfx_funcs;
2130 adev->gfx.config.max_hw_contexts = 8;
2131 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2132 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2133 adev->gfx.config.sc_hiz_tile_fifo_size = 0x30;
2134 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2135 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2136 gb_addr_config &= ~0xf3e777ff;
2137 gb_addr_config |= 0x22014042;
2140 adev->gfx.config.max_hw_contexts = 8;
2141 adev->gfx.config.sc_prim_fifo_size_frontend = 0x20;
2142 adev->gfx.config.sc_prim_fifo_size_backend = 0x100;
2143 adev->gfx.config.sc_hiz_tile_fifo_size = 0x80;
2144 adev->gfx.config.sc_earlyz_tile_fifo_size = 0x4C0;
2145 gb_addr_config = RREG32_SOC15(GC, 0, mmGB_ADDR_CONFIG);
2146 gb_addr_config &= ~0xf3e777ff;
2147 gb_addr_config |= 0x22010042;
2154 adev->gfx.config.gb_addr_config = gb_addr_config;
2156 adev->gfx.config.gb_addr_config_fields.num_pipes = 1 <<
2158 adev->gfx.config.gb_addr_config,
2162 adev->gfx.config.max_tile_pipes =
2163 adev->gfx.config.gb_addr_config_fields.num_pipes;
2165 adev->gfx.config.gb_addr_config_fields.num_banks = 1 <<
2167 adev->gfx.config.gb_addr_config,
2170 adev->gfx.config.gb_addr_config_fields.max_compress_frags = 1 <<
2172 adev->gfx.config.gb_addr_config,
2174 MAX_COMPRESSED_FRAGS);
2175 adev->gfx.config.gb_addr_config_fields.num_rb_per_se = 1 <<
2177 adev->gfx.config.gb_addr_config,
2180 adev->gfx.config.gb_addr_config_fields.num_se = 1 <<
2182 adev->gfx.config.gb_addr_config,
2184 NUM_SHADER_ENGINES);
2185 adev->gfx.config.gb_addr_config_fields.pipe_interleave_size = 1 << (8 +
2187 adev->gfx.config.gb_addr_config,
2189 PIPE_INTERLEAVE_SIZE));
2194 static int gfx_v9_0_compute_ring_init(struct amdgpu_device *adev, int ring_id,
2195 int mec, int pipe, int queue)
2199 struct amdgpu_ring *ring = &adev->gfx.compute_ring[ring_id];
2201 ring = &adev->gfx.compute_ring[ring_id];
2206 ring->queue = queue;
2208 ring->ring_obj = NULL;
2209 ring->use_doorbell = true;
2210 ring->doorbell_index = (adev->doorbell_index.mec_ring0 + ring_id) << 1;
2211 ring->eop_gpu_addr = adev->gfx.mec.hpd_eop_gpu_addr
2212 + (ring_id * GFX9_MEC_HPD_SIZE);
2213 sprintf(ring->name, "comp_%d.%d.%d", ring->me, ring->pipe, ring->queue);
2215 irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP
2216 + ((ring->me - 1) * adev->gfx.mec.num_pipe_per_mec)
2219 /* type-2 packets are deprecated on MEC, use type-3 instead */
2220 r = amdgpu_ring_init(adev, ring, 1024,
2221 &adev->gfx.eop_irq, irq_type);
2229 static int gfx_v9_0_sw_init(void *handle)
2231 int i, j, k, r, ring_id;
2232 struct amdgpu_ring *ring;
2233 struct amdgpu_kiq *kiq;
2234 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2236 switch (adev->asic_type) {
2243 adev->gfx.mec.num_mec = 2;
2246 adev->gfx.mec.num_mec = 1;
2250 adev->gfx.mec.num_pipe_per_mec = 4;
2251 adev->gfx.mec.num_queue_per_pipe = 8;
2254 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_EOP_INTERRUPT, &adev->gfx.eop_irq);
2258 /* Privileged reg */
2259 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_REG_FAULT,
2260 &adev->gfx.priv_reg_irq);
2264 /* Privileged inst */
2265 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_PRIV_INSTR_FAULT,
2266 &adev->gfx.priv_inst_irq);
2271 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_ECC_ERROR,
2272 &adev->gfx.cp_ecc_error_irq);
2277 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_GRBM_CP, GFX_9_0__SRCID__CP_FUE_ERROR,
2278 &adev->gfx.cp_ecc_error_irq);
2282 adev->gfx.gfx_current_status = AMDGPU_GFX_NORMAL_MODE;
2284 gfx_v9_0_scratch_init(adev);
2286 r = gfx_v9_0_init_microcode(adev);
2288 DRM_ERROR("Failed to load gfx firmware!\n");
2292 r = adev->gfx.rlc.funcs->init(adev);
2294 DRM_ERROR("Failed to init rlc BOs!\n");
2298 r = gfx_v9_0_mec_init(adev);
2300 DRM_ERROR("Failed to init MEC BOs!\n");
2304 /* set up the gfx ring */
2305 for (i = 0; i < adev->gfx.num_gfx_rings; i++) {
2306 ring = &adev->gfx.gfx_ring[i];
2307 ring->ring_obj = NULL;
2309 sprintf(ring->name, "gfx");
2311 sprintf(ring->name, "gfx_%d", i);
2312 ring->use_doorbell = true;
2313 ring->doorbell_index = adev->doorbell_index.gfx_ring0 << 1;
2314 r = amdgpu_ring_init(adev, ring, 1024,
2315 &adev->gfx.eop_irq, AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP);
2320 /* set up the compute queues - allocate horizontally across pipes */
2322 for (i = 0; i < adev->gfx.mec.num_mec; ++i) {
2323 for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) {
2324 for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; k++) {
2325 if (!amdgpu_gfx_is_mec_queue_enabled(adev, i, k, j))
2328 r = gfx_v9_0_compute_ring_init(adev,
2339 r = amdgpu_gfx_kiq_init(adev, GFX9_MEC_HPD_SIZE);
2341 DRM_ERROR("Failed to init KIQ BOs!\n");
2345 kiq = &adev->gfx.kiq;
2346 r = amdgpu_gfx_kiq_init_ring(adev, &kiq->ring, &kiq->irq);
2350 /* create MQD for all compute queues as wel as KIQ for SRIOV case */
2351 r = amdgpu_gfx_mqd_sw_init(adev, sizeof(struct v9_mqd_allocation));
2355 adev->gfx.ce_ram_size = 0x8000;
2357 r = gfx_v9_0_gpu_early_init(adev);
2365 static int gfx_v9_0_sw_fini(void *handle)
2368 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2370 amdgpu_gfx_ras_fini(adev);
2372 for (i = 0; i < adev->gfx.num_gfx_rings; i++)
2373 amdgpu_ring_fini(&adev->gfx.gfx_ring[i]);
2374 for (i = 0; i < adev->gfx.num_compute_rings; i++)
2375 amdgpu_ring_fini(&adev->gfx.compute_ring[i]);
2377 amdgpu_gfx_mqd_sw_fini(adev);
2378 amdgpu_gfx_kiq_free_ring(&adev->gfx.kiq.ring);
2379 amdgpu_gfx_kiq_fini(adev);
2381 gfx_v9_0_mec_fini(adev);
2382 amdgpu_bo_unref(&adev->gfx.rlc.clear_state_obj);
2383 if (adev->asic_type == CHIP_RAVEN || adev->asic_type == CHIP_RENOIR) {
2384 amdgpu_bo_free_kernel(&adev->gfx.rlc.cp_table_obj,
2385 &adev->gfx.rlc.cp_table_gpu_addr,
2386 (void **)&adev->gfx.rlc.cp_table_ptr);
2388 gfx_v9_0_free_microcode(adev);
2394 static void gfx_v9_0_tiling_mode_table_init(struct amdgpu_device *adev)
2399 static void gfx_v9_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num, u32 instance)
2403 if (instance == 0xffffffff)
2404 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1);
2406 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance);
2408 if (se_num == 0xffffffff)
2409 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_BROADCAST_WRITES, 1);
2411 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SE_INDEX, se_num);
2413 if (sh_num == 0xffffffff)
2414 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1);
2416 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_INDEX, sh_num);
2418 WREG32_SOC15_RLC_SHADOW(GC, 0, mmGRBM_GFX_INDEX, data);
2421 static u32 gfx_v9_0_get_rb_active_bitmap(struct amdgpu_device *adev)
2425 data = RREG32_SOC15(GC, 0, mmCC_RB_BACKEND_DISABLE);
2426 data |= RREG32_SOC15(GC, 0, mmGC_USER_RB_BACKEND_DISABLE);
2428 data &= CC_RB_BACKEND_DISABLE__BACKEND_DISABLE_MASK;
2429 data >>= GC_USER_RB_BACKEND_DISABLE__BACKEND_DISABLE__SHIFT;
2431 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_backends_per_se /
2432 adev->gfx.config.max_sh_per_se);
2434 return (~data) & mask;
2437 static void gfx_v9_0_setup_rb(struct amdgpu_device *adev)
2442 u32 rb_bitmap_width_per_sh = adev->gfx.config.max_backends_per_se /
2443 adev->gfx.config.max_sh_per_se;
2445 mutex_lock(&adev->grbm_idx_mutex);
2446 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
2447 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
2448 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
2449 data = gfx_v9_0_get_rb_active_bitmap(adev);
2450 active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) *
2451 rb_bitmap_width_per_sh);
2454 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
2455 mutex_unlock(&adev->grbm_idx_mutex);
2457 adev->gfx.config.backend_enable_mask = active_rbs;
2458 adev->gfx.config.num_rbs = hweight32(active_rbs);
2461 #define DEFAULT_SH_MEM_BASES (0x6000)
2462 #define FIRST_COMPUTE_VMID (8)
2463 #define LAST_COMPUTE_VMID (16)
2464 static void gfx_v9_0_init_compute_vmid(struct amdgpu_device *adev)
2467 uint32_t sh_mem_config;
2468 uint32_t sh_mem_bases;
2471 * Configure apertures:
2472 * LDS: 0x60000000'00000000 - 0x60000001'00000000 (4GB)
2473 * Scratch: 0x60000001'00000000 - 0x60000002'00000000 (4GB)
2474 * GPUVM: 0x60010000'00000000 - 0x60020000'00000000 (1TB)
2476 sh_mem_bases = DEFAULT_SH_MEM_BASES | (DEFAULT_SH_MEM_BASES << 16);
2478 sh_mem_config = SH_MEM_ADDRESS_MODE_64 |
2479 SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
2480 SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
2482 mutex_lock(&adev->srbm_mutex);
2483 for (i = FIRST_COMPUTE_VMID; i < LAST_COMPUTE_VMID; i++) {
2484 soc15_grbm_select(adev, 0, 0, 0, i);
2485 /* CP and shaders */
2486 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, sh_mem_config);
2487 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, sh_mem_bases);
2489 soc15_grbm_select(adev, 0, 0, 0, 0);
2490 mutex_unlock(&adev->srbm_mutex);
2492 /* Initialize all compute VMIDs to have no GDS, GWS, or OA
2493 acccess. These should be enabled by FW for target VMIDs. */
2494 for (i = FIRST_COMPUTE_VMID; i < LAST_COMPUTE_VMID; i++) {
2495 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * i, 0);
2496 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * i, 0);
2497 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, i, 0);
2498 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, i, 0);
2502 static void gfx_v9_0_init_gds_vmid(struct amdgpu_device *adev)
2507 * Initialize all compute and user-gfx VMIDs to have no GDS, GWS, or OA
2508 * access. Compute VMIDs should be enabled by FW for target VMIDs,
2509 * the driver can enable them for graphics. VMID0 should maintain
2510 * access so that HWS firmware can save/restore entries.
2512 for (vmid = 1; vmid < 16; vmid++) {
2513 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_BASE, 2 * vmid, 0);
2514 WREG32_SOC15_OFFSET(GC, 0, mmGDS_VMID0_SIZE, 2 * vmid, 0);
2515 WREG32_SOC15_OFFSET(GC, 0, mmGDS_GWS_VMID0, vmid, 0);
2516 WREG32_SOC15_OFFSET(GC, 0, mmGDS_OA_VMID0, vmid, 0);
2520 static void gfx_v9_0_init_sq_config(struct amdgpu_device *adev)
2524 switch (adev->asic_type) {
2526 tmp = RREG32_SOC15(GC, 0, mmSQ_CONFIG);
2527 tmp = REG_SET_FIELD(tmp, SQ_CONFIG,
2528 DISABLE_BARRIER_WAITCNT, 1);
2529 WREG32_SOC15(GC, 0, mmSQ_CONFIG, tmp);
2536 static void gfx_v9_0_constants_init(struct amdgpu_device *adev)
2541 WREG32_FIELD15_RLC(GC, 0, GRBM_CNTL, READ_TIMEOUT, 0xff);
2543 gfx_v9_0_tiling_mode_table_init(adev);
2545 gfx_v9_0_setup_rb(adev);
2546 gfx_v9_0_get_cu_info(adev, &adev->gfx.cu_info);
2547 adev->gfx.config.db_debug2 = RREG32_SOC15(GC, 0, mmDB_DEBUG2);
2549 /* XXX SH_MEM regs */
2550 /* where to put LDS, scratch, GPUVM in FSA64 space */
2551 mutex_lock(&adev->srbm_mutex);
2552 for (i = 0; i < adev->vm_manager.id_mgr[AMDGPU_GFXHUB_0].num_ids; i++) {
2553 soc15_grbm_select(adev, 0, 0, 0, i);
2554 /* CP and shaders */
2556 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
2557 SH_MEM_ALIGNMENT_MODE_UNALIGNED);
2558 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
2560 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);
2561 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, 0);
2563 tmp = REG_SET_FIELD(0, SH_MEM_CONFIG, ALIGNMENT_MODE,
2564 SH_MEM_ALIGNMENT_MODE_UNALIGNED);
2565 tmp = REG_SET_FIELD(tmp, SH_MEM_CONFIG, RETRY_DISABLE,
2567 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_CONFIG, tmp);
2568 tmp = REG_SET_FIELD(0, SH_MEM_BASES, PRIVATE_BASE,
2569 (adev->gmc.private_aperture_start >> 48));
2570 tmp = REG_SET_FIELD(tmp, SH_MEM_BASES, SHARED_BASE,
2571 (adev->gmc.shared_aperture_start >> 48));
2572 WREG32_SOC15_RLC(GC, 0, mmSH_MEM_BASES, tmp);
2575 soc15_grbm_select(adev, 0, 0, 0, 0);
2577 mutex_unlock(&adev->srbm_mutex);
2579 gfx_v9_0_init_compute_vmid(adev);
2580 gfx_v9_0_init_gds_vmid(adev);
2581 gfx_v9_0_init_sq_config(adev);
2584 static void gfx_v9_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
2589 mutex_lock(&adev->grbm_idx_mutex);
2590 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
2591 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
2592 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
2593 for (k = 0; k < adev->usec_timeout; k++) {
2594 if (RREG32_SOC15(GC, 0, mmRLC_SERDES_CU_MASTER_BUSY) == 0)
2598 if (k == adev->usec_timeout) {
2599 gfx_v9_0_select_se_sh(adev, 0xffffffff,
2600 0xffffffff, 0xffffffff);
2601 mutex_unlock(&adev->grbm_idx_mutex);
2602 DRM_INFO("Timeout wait for RLC serdes %u,%u\n",
2608 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
2609 mutex_unlock(&adev->grbm_idx_mutex);
2611 mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK |
2612 RLC_SERDES_NONCU_MASTER_BUSY__GC_MASTER_BUSY_MASK |
2613 RLC_SERDES_NONCU_MASTER_BUSY__TC0_MASTER_BUSY_MASK |
2614 RLC_SERDES_NONCU_MASTER_BUSY__TC1_MASTER_BUSY_MASK;
2615 for (k = 0; k < adev->usec_timeout; k++) {
2616 if ((RREG32_SOC15(GC, 0, mmRLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
2622 static void gfx_v9_0_enable_gui_idle_interrupt(struct amdgpu_device *adev,
2625 u32 tmp = RREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0);
2627 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE, enable ? 1 : 0);
2628 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CNTX_EMPTY_INT_ENABLE, enable ? 1 : 0);
2629 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, CMP_BUSY_INT_ENABLE, enable ? 1 : 0);
2630 tmp = REG_SET_FIELD(tmp, CP_INT_CNTL_RING0, GFX_IDLE_INT_ENABLE, enable ? 1 : 0);
2632 WREG32_SOC15(GC, 0, mmCP_INT_CNTL_RING0, tmp);
2635 static void gfx_v9_0_init_csb(struct amdgpu_device *adev)
2637 adev->gfx.rlc.funcs->get_csb_buffer(adev, adev->gfx.rlc.cs_ptr);
2639 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_HI),
2640 adev->gfx.rlc.clear_state_gpu_addr >> 32);
2641 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_ADDR_LO),
2642 adev->gfx.rlc.clear_state_gpu_addr & 0xfffffffc);
2643 WREG32_RLC(SOC15_REG_OFFSET(GC, 0, mmRLC_CSIB_LENGTH),
2644 adev->gfx.rlc.clear_state_size);
2647 static void gfx_v9_1_parse_ind_reg_list(int *register_list_format,
2648 int indirect_offset,
2650 int *unique_indirect_regs,
2651 int unique_indirect_reg_count,
2652 int *indirect_start_offsets,
2653 int *indirect_start_offsets_count,
2654 int max_start_offsets_count)
2658 for (; indirect_offset < list_size; indirect_offset++) {
2659 WARN_ON(*indirect_start_offsets_count >= max_start_offsets_count);
2660 indirect_start_offsets[*indirect_start_offsets_count] = indirect_offset;
2661 *indirect_start_offsets_count = *indirect_start_offsets_count + 1;
2663 while (register_list_format[indirect_offset] != 0xFFFFFFFF) {
2664 indirect_offset += 2;
2666 /* look for the matching indice */
2667 for (idx = 0; idx < unique_indirect_reg_count; idx++) {
2668 if (unique_indirect_regs[idx] ==
2669 register_list_format[indirect_offset] ||
2670 !unique_indirect_regs[idx])
2674 BUG_ON(idx >= unique_indirect_reg_count);
2676 if (!unique_indirect_regs[idx])
2677 unique_indirect_regs[idx] = register_list_format[indirect_offset];
2684 static int gfx_v9_1_init_rlc_save_restore_list(struct amdgpu_device *adev)
2686 int unique_indirect_regs[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
2687 int unique_indirect_reg_count = 0;
2689 int indirect_start_offsets[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0};
2690 int indirect_start_offsets_count = 0;
2696 u32 *register_list_format =
2697 kmemdup(adev->gfx.rlc.register_list_format,
2698 adev->gfx.rlc.reg_list_format_size_bytes, GFP_KERNEL);
2699 if (!register_list_format)
2702 /* setup unique_indirect_regs array and indirect_start_offsets array */
2703 unique_indirect_reg_count = ARRAY_SIZE(unique_indirect_regs);
2704 gfx_v9_1_parse_ind_reg_list(register_list_format,
2705 adev->gfx.rlc.reg_list_format_direct_reg_list_length,
2706 adev->gfx.rlc.reg_list_format_size_bytes >> 2,
2707 unique_indirect_regs,
2708 unique_indirect_reg_count,
2709 indirect_start_offsets,
2710 &indirect_start_offsets_count,
2711 ARRAY_SIZE(indirect_start_offsets));
2713 /* enable auto inc in case it is disabled */
2714 tmp = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL));
2715 tmp |= RLC_SRM_CNTL__AUTO_INCR_ADDR_MASK;
2716 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_CNTL), tmp);
2718 /* write register_restore table to offset 0x0 using RLC_SRM_ARAM_ADDR/DATA */
2719 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_ADDR),
2720 RLC_SAVE_RESTORE_ADDR_STARTING_OFFSET);
2721 for (i = 0; i < adev->gfx.rlc.reg_list_size_bytes >> 2; i++)
2722 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_ARAM_DATA),
2723 adev->gfx.rlc.register_restore[i]);
2725 /* load indirect register */
2726 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2727 adev->gfx.rlc.reg_list_format_start);
2729 /* direct register portion */
2730 for (i = 0; i < adev->gfx.rlc.reg_list_format_direct_reg_list_length; i++)
2731 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
2732 register_list_format[i]);
2734 /* indirect register portion */
2735 while (i < (adev->gfx.rlc.reg_list_format_size_bytes >> 2)) {
2736 if (register_list_format[i] == 0xFFFFFFFF) {
2737 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2741 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2742 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, register_list_format[i++]);
2744 for (j = 0; j < unique_indirect_reg_count; j++) {
2745 if (register_list_format[i] == unique_indirect_regs[j]) {
2746 WREG32_SOC15(GC, 0, mmRLC_GPM_SCRATCH_DATA, j);
2751 BUG_ON(j >= unique_indirect_reg_count);
2756 /* set save/restore list size */
2757 list_size = adev->gfx.rlc.reg_list_size_bytes >> 2;
2758 list_size = list_size >> 1;
2759 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2760 adev->gfx.rlc.reg_restore_list_size);
2761 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA), list_size);
2763 /* write the starting offsets to RLC scratch ram */
2764 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_ADDR),
2765 adev->gfx.rlc.starting_offsets_start);
2766 for (i = 0; i < ARRAY_SIZE(indirect_start_offsets); i++)
2767 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_GPM_SCRATCH_DATA),
2768 indirect_start_offsets[i]);
2770 /* load unique indirect regs*/
2771 for (i = 0; i < ARRAY_SIZE(unique_indirect_regs); i++) {
2772 if (unique_indirect_regs[i] != 0) {
2773 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_ADDR_0)
2774 + GFX_RLC_SRM_INDEX_CNTL_ADDR_OFFSETS[i],
2775 unique_indirect_regs[i] & 0x3FFFF);
2777 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_SRM_INDEX_CNTL_DATA_0)
2778 + GFX_RLC_SRM_INDEX_CNTL_DATA_OFFSETS[i],
2779 unique_indirect_regs[i] >> 20);
2783 kfree(register_list_format);
2787 static void gfx_v9_0_enable_save_restore_machine(struct amdgpu_device *adev)
2789 WREG32_FIELD15(GC, 0, RLC_SRM_CNTL, SRM_ENABLE, 1);
2792 static void pwr_10_0_gfxip_control_over_cgpg(struct amdgpu_device *adev,
2796 uint32_t default_data = 0;
2798 default_data = data = RREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS));
2799 if (enable == true) {
2800 /* enable GFXIP control over CGPG */
2801 data |= PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK;
2802 if(default_data != data)
2803 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2806 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS_MASK;
2807 data |= (2 << PWR_MISC_CNTL_STATUS__PWR_GFXOFF_STATUS__SHIFT);
2808 if(default_data != data)
2809 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2811 /* restore GFXIP control over GCPG */
2812 data &= ~PWR_MISC_CNTL_STATUS__PWR_GFX_RLC_CGPG_EN_MASK;
2813 if(default_data != data)
2814 WREG32(SOC15_REG_OFFSET(PWR, 0, mmPWR_MISC_CNTL_STATUS), data);
2818 static void gfx_v9_0_init_gfx_power_gating(struct amdgpu_device *adev)
2822 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
2823 AMD_PG_SUPPORT_GFX_SMG |
2824 AMD_PG_SUPPORT_GFX_DMG)) {
2825 /* init IDLE_POLL_COUNT = 60 */
2826 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL));
2827 data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK;
2828 data |= (0x60 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
2829 WREG32(SOC15_REG_OFFSET(GC, 0, mmCP_RB_WPTR_POLL_CNTL), data);
2831 /* init RLC PG Delay */
2833 data |= (0x10 << RLC_PG_DELAY__POWER_UP_DELAY__SHIFT);
2834 data |= (0x10 << RLC_PG_DELAY__POWER_DOWN_DELAY__SHIFT);
2835 data |= (0x10 << RLC_PG_DELAY__CMD_PROPAGATE_DELAY__SHIFT);
2836 data |= (0x40 << RLC_PG_DELAY__MEM_SLEEP_DELAY__SHIFT);
2837 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY), data);
2839 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2));
2840 data &= ~RLC_PG_DELAY_2__SERDES_CMD_DELAY_MASK;
2841 data |= (0x4 << RLC_PG_DELAY_2__SERDES_CMD_DELAY__SHIFT);
2842 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_2), data);
2844 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3));
2845 data &= ~RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK;
2846 data |= (0xff << RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT);
2847 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_DELAY_3), data);
2849 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL));
2850 data &= ~RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD_MASK;
2852 /* program GRBM_REG_SAVE_GFX_IDLE_THRESHOLD to 0x55f0 */
2853 data |= (0x55f0 << RLC_AUTO_PG_CTRL__GRBM_REG_SAVE_GFX_IDLE_THRESHOLD__SHIFT);
2854 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_AUTO_PG_CTRL), data);
2856 pwr_10_0_gfxip_control_over_cgpg(adev, true);
2860 static void gfx_v9_0_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev,
2864 uint32_t default_data = 0;
2866 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2867 data = REG_SET_FIELD(data, RLC_PG_CNTL,
2868 SMU_CLK_SLOWDOWN_ON_PU_ENABLE,
2870 if (default_data != data)
2871 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2874 static void gfx_v9_0_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev,
2878 uint32_t default_data = 0;
2880 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2881 data = REG_SET_FIELD(data, RLC_PG_CNTL,
2882 SMU_CLK_SLOWDOWN_ON_PD_ENABLE,
2884 if(default_data != data)
2885 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2888 static void gfx_v9_0_enable_cp_power_gating(struct amdgpu_device *adev,
2892 uint32_t default_data = 0;
2894 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2895 data = REG_SET_FIELD(data, RLC_PG_CNTL,
2898 if(default_data != data)
2899 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2902 static void gfx_v9_0_enable_gfx_cg_power_gating(struct amdgpu_device *adev,
2905 uint32_t data, default_data;
2907 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2908 data = REG_SET_FIELD(data, RLC_PG_CNTL,
2909 GFX_POWER_GATING_ENABLE,
2911 if(default_data != data)
2912 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2915 static void gfx_v9_0_enable_gfx_pipeline_powergating(struct amdgpu_device *adev,
2918 uint32_t data, default_data;
2920 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2921 data = REG_SET_FIELD(data, RLC_PG_CNTL,
2922 GFX_PIPELINE_PG_ENABLE,
2924 if(default_data != data)
2925 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2928 /* read any GFX register to wake up GFX */
2929 data = RREG32(SOC15_REG_OFFSET(GC, 0, mmDB_RENDER_CONTROL));
2932 static void gfx_v9_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev,
2935 uint32_t data, default_data;
2937 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2938 data = REG_SET_FIELD(data, RLC_PG_CNTL,
2939 STATIC_PER_CU_PG_ENABLE,
2941 if(default_data != data)
2942 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2945 static void gfx_v9_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev,
2948 uint32_t data, default_data;
2950 default_data = data = RREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL));
2951 data = REG_SET_FIELD(data, RLC_PG_CNTL,
2952 DYN_PER_CU_PG_ENABLE,
2954 if(default_data != data)
2955 WREG32(SOC15_REG_OFFSET(GC, 0, mmRLC_PG_CNTL), data);
2958 static void gfx_v9_0_init_pg(struct amdgpu_device *adev)
2960 gfx_v9_0_init_csb(adev);
2963 * Rlc save restore list is workable since v2_1.
2964 * And it's needed by gfxoff feature.
2966 if (adev->gfx.rlc.is_rlc_v2_1) {
2967 if (adev->asic_type == CHIP_VEGA12 ||
2968 (adev->asic_type == CHIP_RAVEN &&
2970 gfx_v9_1_init_rlc_save_restore_list(adev);
2971 gfx_v9_0_enable_save_restore_machine(adev);
2974 if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
2975 AMD_PG_SUPPORT_GFX_SMG |
2976 AMD_PG_SUPPORT_GFX_DMG |
2978 AMD_PG_SUPPORT_GDS |
2979 AMD_PG_SUPPORT_RLC_SMU_HS)) {
2980 WREG32(mmRLC_JUMP_TABLE_RESTORE,
2981 adev->gfx.rlc.cp_table_gpu_addr >> 8);
2982 gfx_v9_0_init_gfx_power_gating(adev);
2986 void gfx_v9_0_rlc_stop(struct amdgpu_device *adev)
2988 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 0);
2989 gfx_v9_0_enable_gui_idle_interrupt(adev, false);
2990 gfx_v9_0_wait_for_rlc_serdes(adev);
2993 static void gfx_v9_0_rlc_reset(struct amdgpu_device *adev)
2995 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
2997 WREG32_FIELD15(GC, 0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 0);
3001 static void gfx_v9_0_rlc_start(struct amdgpu_device *adev)
3003 #ifdef AMDGPU_RLC_DEBUG_RETRY
3007 WREG32_FIELD15(GC, 0, RLC_CNTL, RLC_ENABLE_F32, 1);
3010 /* carrizo do enable cp interrupt after cp inited */
3011 if (!(adev->flags & AMD_IS_APU)) {
3012 gfx_v9_0_enable_gui_idle_interrupt(adev, true);
3016 #ifdef AMDGPU_RLC_DEBUG_RETRY
3017 /* RLC_GPM_GENERAL_6 : RLC Ucode version */
3018 rlc_ucode_ver = RREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_6);
3019 if(rlc_ucode_ver == 0x108) {
3020 DRM_INFO("Using rlc debug ucode. mmRLC_GPM_GENERAL_6 ==0x08%x / fw_ver == %i \n",
3021 rlc_ucode_ver, adev->gfx.rlc_fw_version);
3022 /* RLC_GPM_TIMER_INT_3 : Timer interval in RefCLK cycles,
3023 * default is 0x9C4 to create a 100us interval */
3024 WREG32_SOC15(GC, 0, mmRLC_GPM_TIMER_INT_3, 0x9C4);
3025 /* RLC_GPM_GENERAL_12 : Minimum gap between wptr and rptr
3026 * to disable the page fault retry interrupts, default is
3028 WREG32_SOC15(GC, 0, mmRLC_GPM_GENERAL_12, 0x100);
3033 static int gfx_v9_0_rlc_load_microcode(struct amdgpu_device *adev)
3035 const struct rlc_firmware_header_v2_0 *hdr;
3036 const __le32 *fw_data;
3037 unsigned i, fw_size;
3039 if (!adev->gfx.rlc_fw)
3042 hdr = (const struct rlc_firmware_header_v2_0 *)adev->gfx.rlc_fw->data;
3043 amdgpu_ucode_print_rlc_hdr(&hdr->header);
3045 fw_data = (const __le32 *)(adev->gfx.rlc_fw->data +
3046 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
3047 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
3049 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR,
3050 RLCG_UCODE_LOADING_START_ADDRESS);
3051 for (i = 0; i < fw_size; i++)
3052 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_DATA, le32_to_cpup(fw_data++));
3053 WREG32_SOC15(GC, 0, mmRLC_GPM_UCODE_ADDR, adev->gfx.rlc_fw_version);
3058 static int gfx_v9_0_rlc_resume(struct amdgpu_device *adev)
3062 if (amdgpu_sriov_vf(adev)) {
3063 gfx_v9_0_init_csb(adev);
3067 adev->gfx.rlc.funcs->stop(adev);
3070 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, 0);
3072 gfx_v9_0_init_pg(adev);
3074 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
3075 /* legacy rlc firmware loading */
3076 r = gfx_v9_0_rlc_load_microcode(adev);
3081 switch (adev->asic_type) {
3083 if (amdgpu_lbpw == 0)
3084 gfx_v9_0_enable_lbpw(adev, false);
3086 gfx_v9_0_enable_lbpw(adev, true);
3089 if (amdgpu_lbpw > 0)
3090 gfx_v9_0_enable_lbpw(adev, true);
3092 gfx_v9_0_enable_lbpw(adev, false);
3098 adev->gfx.rlc.funcs->start(adev);
3103 static void gfx_v9_0_cp_gfx_enable(struct amdgpu_device *adev, bool enable)
3106 u32 tmp = RREG32_SOC15(GC, 0, mmCP_ME_CNTL);
3108 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, ME_HALT, enable ? 0 : 1);
3109 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, PFP_HALT, enable ? 0 : 1);
3110 tmp = REG_SET_FIELD(tmp, CP_ME_CNTL, CE_HALT, enable ? 0 : 1);
3112 for (i = 0; i < adev->gfx.num_gfx_rings; i++)
3113 adev->gfx.gfx_ring[i].sched.ready = false;
3115 WREG32_SOC15_RLC(GC, 0, mmCP_ME_CNTL, tmp);
3119 static int gfx_v9_0_cp_gfx_load_microcode(struct amdgpu_device *adev)
3121 const struct gfx_firmware_header_v1_0 *pfp_hdr;
3122 const struct gfx_firmware_header_v1_0 *ce_hdr;
3123 const struct gfx_firmware_header_v1_0 *me_hdr;
3124 const __le32 *fw_data;
3125 unsigned i, fw_size;
3127 if (!adev->gfx.me_fw || !adev->gfx.pfp_fw || !adev->gfx.ce_fw)
3130 pfp_hdr = (const struct gfx_firmware_header_v1_0 *)
3131 adev->gfx.pfp_fw->data;
3132 ce_hdr = (const struct gfx_firmware_header_v1_0 *)
3133 adev->gfx.ce_fw->data;
3134 me_hdr = (const struct gfx_firmware_header_v1_0 *)
3135 adev->gfx.me_fw->data;
3137 amdgpu_ucode_print_gfx_hdr(&pfp_hdr->header);
3138 amdgpu_ucode_print_gfx_hdr(&ce_hdr->header);
3139 amdgpu_ucode_print_gfx_hdr(&me_hdr->header);
3141 gfx_v9_0_cp_gfx_enable(adev, false);
3144 fw_data = (const __le32 *)
3145 (adev->gfx.pfp_fw->data +
3146 le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3147 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3148 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, 0);
3149 for (i = 0; i < fw_size; i++)
3150 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
3151 WREG32_SOC15(GC, 0, mmCP_PFP_UCODE_ADDR, adev->gfx.pfp_fw_version);
3154 fw_data = (const __le32 *)
3155 (adev->gfx.ce_fw->data +
3156 le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3157 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3158 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, 0);
3159 for (i = 0; i < fw_size; i++)
3160 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
3161 WREG32_SOC15(GC, 0, mmCP_CE_UCODE_ADDR, adev->gfx.ce_fw_version);
3164 fw_data = (const __le32 *)
3165 (adev->gfx.me_fw->data +
3166 le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3167 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3168 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, 0);
3169 for (i = 0; i < fw_size; i++)
3170 WREG32_SOC15(GC, 0, mmCP_ME_RAM_DATA, le32_to_cpup(fw_data++));
3171 WREG32_SOC15(GC, 0, mmCP_ME_RAM_WADDR, adev->gfx.me_fw_version);
3176 static int gfx_v9_0_cp_gfx_start(struct amdgpu_device *adev)
3178 struct amdgpu_ring *ring = &adev->gfx.gfx_ring[0];
3179 const struct cs_section_def *sect = NULL;
3180 const struct cs_extent_def *ext = NULL;
3184 WREG32_SOC15(GC, 0, mmCP_MAX_CONTEXT, adev->gfx.config.max_hw_contexts - 1);
3185 WREG32_SOC15(GC, 0, mmCP_DEVICE_ID, 1);
3187 gfx_v9_0_cp_gfx_enable(adev, true);
3189 r = amdgpu_ring_alloc(ring, gfx_v9_0_get_csb_size(adev) + 4 + 3);
3191 DRM_ERROR("amdgpu: cp failed to lock ring (%d).\n", r);
3195 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3196 amdgpu_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3198 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3199 amdgpu_ring_write(ring, 0x80000000);
3200 amdgpu_ring_write(ring, 0x80000000);
3202 for (sect = gfx9_cs_data; sect->section != NULL; ++sect) {
3203 for (ext = sect->section; ext->extent != NULL; ++ext) {
3204 if (sect->id == SECT_CONTEXT) {
3205 amdgpu_ring_write(ring,
3206 PACKET3(PACKET3_SET_CONTEXT_REG,
3208 amdgpu_ring_write(ring,
3209 ext->reg_index - PACKET3_SET_CONTEXT_REG_START);
3210 for (i = 0; i < ext->reg_count; i++)
3211 amdgpu_ring_write(ring, ext->extent[i]);
3216 amdgpu_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3217 amdgpu_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3219 amdgpu_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3220 amdgpu_ring_write(ring, 0);
3222 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3223 amdgpu_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3224 amdgpu_ring_write(ring, 0x8000);
3225 amdgpu_ring_write(ring, 0x8000);
3227 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG,1));
3228 tmp = (PACKET3_SET_UCONFIG_REG_INDEX_TYPE |
3229 (SOC15_REG_OFFSET(GC, 0, mmVGT_INDEX_TYPE) - PACKET3_SET_UCONFIG_REG_START));
3230 amdgpu_ring_write(ring, tmp);
3231 amdgpu_ring_write(ring, 0);
3233 amdgpu_ring_commit(ring);
3238 static int gfx_v9_0_cp_gfx_resume(struct amdgpu_device *adev)
3240 struct amdgpu_ring *ring;
3243 u64 rb_addr, rptr_addr, wptr_gpu_addr;
3245 /* Set the write pointer delay */
3246 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_DELAY, 0);
3248 /* set the RB to use vmid 0 */
3249 WREG32_SOC15(GC, 0, mmCP_RB_VMID, 0);
3251 /* Set ring buffer size */
3252 ring = &adev->gfx.gfx_ring[0];
3253 rb_bufsz = order_base_2(ring->ring_size / 8);
3254 tmp = REG_SET_FIELD(0, CP_RB0_CNTL, RB_BUFSZ, rb_bufsz);
3255 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, RB_BLKSZ, rb_bufsz - 2);
3257 tmp = REG_SET_FIELD(tmp, CP_RB0_CNTL, BUF_SWAP, 1);
3259 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
3261 /* Initialize the ring buffer's write pointers */
3263 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
3264 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
3266 /* set the wb address wether it's enabled or not */
3267 rptr_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
3268 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR, lower_32_bits(rptr_addr));
3269 WREG32_SOC15(GC, 0, mmCP_RB0_RPTR_ADDR_HI, upper_32_bits(rptr_addr) & CP_RB_RPTR_ADDR_HI__RB_RPTR_ADDR_HI_MASK);
3271 wptr_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
3272 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_LO, lower_32_bits(wptr_gpu_addr));
3273 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_ADDR_HI, upper_32_bits(wptr_gpu_addr));
3276 WREG32_SOC15(GC, 0, mmCP_RB0_CNTL, tmp);
3278 rb_addr = ring->gpu_addr >> 8;
3279 WREG32_SOC15(GC, 0, mmCP_RB0_BASE, rb_addr);
3280 WREG32_SOC15(GC, 0, mmCP_RB0_BASE_HI, upper_32_bits(rb_addr));
3282 tmp = RREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL);
3283 if (ring->use_doorbell) {
3284 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3285 DOORBELL_OFFSET, ring->doorbell_index);
3286 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL,
3289 tmp = REG_SET_FIELD(tmp, CP_RB_DOORBELL_CONTROL, DOORBELL_EN, 0);
3291 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_CONTROL, tmp);
3293 tmp = REG_SET_FIELD(0, CP_RB_DOORBELL_RANGE_LOWER,
3294 DOORBELL_RANGE_LOWER, ring->doorbell_index);
3295 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_LOWER, tmp);
3297 WREG32_SOC15(GC, 0, mmCP_RB_DOORBELL_RANGE_UPPER,
3298 CP_RB_DOORBELL_RANGE_UPPER__DOORBELL_RANGE_UPPER_MASK);
3301 /* start the ring */
3302 gfx_v9_0_cp_gfx_start(adev);
3303 ring->sched.ready = true;
3308 static void gfx_v9_0_cp_compute_enable(struct amdgpu_device *adev, bool enable)
3313 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL, 0);
3315 WREG32_SOC15_RLC(GC, 0, mmCP_MEC_CNTL,
3316 (CP_MEC_CNTL__MEC_ME1_HALT_MASK | CP_MEC_CNTL__MEC_ME2_HALT_MASK));
3317 for (i = 0; i < adev->gfx.num_compute_rings; i++)
3318 adev->gfx.compute_ring[i].sched.ready = false;
3319 adev->gfx.kiq.ring.sched.ready = false;
3324 static int gfx_v9_0_cp_compute_load_microcode(struct amdgpu_device *adev)
3326 const struct gfx_firmware_header_v1_0 *mec_hdr;
3327 const __le32 *fw_data;
3331 if (!adev->gfx.mec_fw)
3334 gfx_v9_0_cp_compute_enable(adev, false);
3336 mec_hdr = (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
3337 amdgpu_ucode_print_gfx_hdr(&mec_hdr->header);
3339 fw_data = (const __le32 *)
3340 (adev->gfx.mec_fw->data +
3341 le32_to_cpu(mec_hdr->header.ucode_array_offset_bytes));
3343 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, VMID, 0);
3344 tmp = REG_SET_FIELD(tmp, CP_CPC_IC_BASE_CNTL, CACHE_POLICY, 0);
3345 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_CNTL, tmp);
3347 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_LO,
3348 adev->gfx.mec.mec_fw_gpu_addr & 0xFFFFF000);
3349 WREG32_SOC15(GC, 0, mmCP_CPC_IC_BASE_HI,
3350 upper_32_bits(adev->gfx.mec.mec_fw_gpu_addr));
3353 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,
3354 mec_hdr->jt_offset);
3355 for (i = 0; i < mec_hdr->jt_size; i++)
3356 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_DATA,
3357 le32_to_cpup(fw_data + mec_hdr->jt_offset + i));
3359 WREG32_SOC15(GC, 0, mmCP_MEC_ME1_UCODE_ADDR,
3360 adev->gfx.mec_fw_version);
3361 /* Todo : Loading MEC2 firmware is only necessary if MEC2 should run different microcode than MEC1. */
3367 static void gfx_v9_0_kiq_setting(struct amdgpu_ring *ring)
3370 struct amdgpu_device *adev = ring->adev;
3372 /* tell RLC which is KIQ queue */
3373 tmp = RREG32_SOC15(GC, 0, mmRLC_CP_SCHEDULERS);
3375 tmp |= (ring->me << 5) | (ring->pipe << 3) | (ring->queue);
3376 WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
3378 WREG32_SOC15_RLC(GC, 0, mmRLC_CP_SCHEDULERS, tmp);
3381 static void gfx_v9_0_mqd_set_priority(struct amdgpu_ring *ring, struct v9_mqd *mqd)
3383 struct amdgpu_device *adev = ring->adev;
3385 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
3386 if (amdgpu_gfx_is_high_priority_compute_queue(adev, ring->queue)) {
3387 mqd->cp_hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_HIGH;
3388 ring->has_high_prio = true;
3389 mqd->cp_hqd_queue_priority =
3390 AMDGPU_GFX_QUEUE_PRIORITY_MAXIMUM;
3392 ring->has_high_prio = false;
3397 static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring)
3399 struct amdgpu_device *adev = ring->adev;
3400 struct v9_mqd *mqd = ring->mqd_ptr;
3401 uint64_t hqd_gpu_addr, wb_gpu_addr, eop_base_addr;
3404 mqd->header = 0xC0310800;
3405 mqd->compute_pipelinestat_enable = 0x00000001;
3406 mqd->compute_static_thread_mgmt_se0 = 0xffffffff;
3407 mqd->compute_static_thread_mgmt_se1 = 0xffffffff;
3408 mqd->compute_static_thread_mgmt_se2 = 0xffffffff;
3409 mqd->compute_static_thread_mgmt_se3 = 0xffffffff;
3410 mqd->compute_static_thread_mgmt_se4 = 0xffffffff;
3411 mqd->compute_static_thread_mgmt_se5 = 0xffffffff;
3412 mqd->compute_static_thread_mgmt_se6 = 0xffffffff;
3413 mqd->compute_static_thread_mgmt_se7 = 0xffffffff;
3414 mqd->compute_misc_reserved = 0x00000003;
3416 mqd->dynamic_cu_mask_addr_lo =
3417 lower_32_bits(ring->mqd_gpu_addr
3418 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
3419 mqd->dynamic_cu_mask_addr_hi =
3420 upper_32_bits(ring->mqd_gpu_addr
3421 + offsetof(struct v9_mqd_allocation, dynamic_cu_mask));
3423 eop_base_addr = ring->eop_gpu_addr >> 8;
3424 mqd->cp_hqd_eop_base_addr_lo = eop_base_addr;
3425 mqd->cp_hqd_eop_base_addr_hi = upper_32_bits(eop_base_addr);
3427 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3428 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_EOP_CONTROL);
3429 tmp = REG_SET_FIELD(tmp, CP_HQD_EOP_CONTROL, EOP_SIZE,
3430 (order_base_2(GFX9_MEC_HPD_SIZE / 4) - 1));
3432 mqd->cp_hqd_eop_control = tmp;
3434 /* enable doorbell? */
3435 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);
3437 if (ring->use_doorbell) {
3438 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3439 DOORBELL_OFFSET, ring->doorbell_index);
3440 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3442 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3443 DOORBELL_SOURCE, 0);
3444 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3447 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3451 mqd->cp_hqd_pq_doorbell_control = tmp;
3453 /* disable the queue if it's active */
3455 mqd->cp_hqd_dequeue_request = 0;
3456 mqd->cp_hqd_pq_rptr = 0;
3457 mqd->cp_hqd_pq_wptr_lo = 0;
3458 mqd->cp_hqd_pq_wptr_hi = 0;
3460 /* set the pointer to the MQD */
3461 mqd->cp_mqd_base_addr_lo = ring->mqd_gpu_addr & 0xfffffffc;
3462 mqd->cp_mqd_base_addr_hi = upper_32_bits(ring->mqd_gpu_addr);
3464 /* set MQD vmid to 0 */
3465 tmp = RREG32_SOC15(GC, 0, mmCP_MQD_CONTROL);
3466 tmp = REG_SET_FIELD(tmp, CP_MQD_CONTROL, VMID, 0);
3467 mqd->cp_mqd_control = tmp;
3469 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3470 hqd_gpu_addr = ring->gpu_addr >> 8;
3471 mqd->cp_hqd_pq_base_lo = hqd_gpu_addr;
3472 mqd->cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
3474 /* set up the HQD, this is similar to CP_RB0_CNTL */
3475 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_CONTROL);
3476 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, QUEUE_SIZE,
3477 (order_base_2(ring->ring_size / 4) - 1));
3478 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, RPTR_BLOCK_SIZE,
3479 ((order_base_2(AMDGPU_GPU_PAGE_SIZE / 4) - 1) << 8));
3481 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ENDIAN_SWAP, 1);
3483 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, UNORD_DISPATCH, 0);
3484 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, ROQ_PQ_IB_FLIP, 0);
3485 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, PRIV_STATE, 1);
3486 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_CONTROL, KMD_QUEUE, 1);
3487 mqd->cp_hqd_pq_control = tmp;
3489 /* set the wb address whether it's enabled or not */
3490 wb_gpu_addr = adev->wb.gpu_addr + (ring->rptr_offs * 4);
3491 mqd->cp_hqd_pq_rptr_report_addr_lo = wb_gpu_addr & 0xfffffffc;
3492 mqd->cp_hqd_pq_rptr_report_addr_hi =
3493 upper_32_bits(wb_gpu_addr) & 0xffff;
3495 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3496 wb_gpu_addr = adev->wb.gpu_addr + (ring->wptr_offs * 4);
3497 mqd->cp_hqd_pq_wptr_poll_addr_lo = wb_gpu_addr & 0xfffffffc;
3498 mqd->cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
3501 /* enable the doorbell if requested */
3502 if (ring->use_doorbell) {
3503 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL);
3504 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3505 DOORBELL_OFFSET, ring->doorbell_index);
3507 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3509 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3510 DOORBELL_SOURCE, 0);
3511 tmp = REG_SET_FIELD(tmp, CP_HQD_PQ_DOORBELL_CONTROL,
3515 mqd->cp_hqd_pq_doorbell_control = tmp;
3517 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3519 mqd->cp_hqd_pq_rptr = RREG32_SOC15(GC, 0, mmCP_HQD_PQ_RPTR);
3521 /* set the vmid for the queue */
3522 mqd->cp_hqd_vmid = 0;
3524 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_PERSISTENT_STATE);
3525 tmp = REG_SET_FIELD(tmp, CP_HQD_PERSISTENT_STATE, PRELOAD_SIZE, 0x53);
3526 mqd->cp_hqd_persistent_state = tmp;
3528 /* set MIN_IB_AVAIL_SIZE */
3529 tmp = RREG32_SOC15(GC, 0, mmCP_HQD_IB_CONTROL);
3530 tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
3531 mqd->cp_hqd_ib_control = tmp;
3533 /* set static priority for a queue/ring */
3534 gfx_v9_0_mqd_set_priority(ring, mqd);
3535 mqd->cp_hqd_quantum = RREG32(mmCP_HQD_QUANTUM);
3537 /* map_queues packet doesn't need activate the queue,
3538 * so only kiq need set this field.
3540 if (ring->funcs->type == AMDGPU_RING_TYPE_KIQ)
3541 mqd->cp_hqd_active = 1;
3546 static int gfx_v9_0_kiq_init_register(struct amdgpu_ring *ring)
3548 struct amdgpu_device *adev = ring->adev;
3549 struct v9_mqd *mqd = ring->mqd_ptr;
3552 /* disable wptr polling */
3553 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
3555 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR,
3556 mqd->cp_hqd_eop_base_addr_lo);
3557 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_BASE_ADDR_HI,
3558 mqd->cp_hqd_eop_base_addr_hi);
3560 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
3561 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_EOP_CONTROL,
3562 mqd->cp_hqd_eop_control);
3564 /* enable doorbell? */
3565 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
3566 mqd->cp_hqd_pq_doorbell_control);
3568 /* disable the queue if it's active */
3569 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
3570 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
3571 for (j = 0; j < adev->usec_timeout; j++) {
3572 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
3576 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
3577 mqd->cp_hqd_dequeue_request);
3578 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR,
3579 mqd->cp_hqd_pq_rptr);
3580 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,
3581 mqd->cp_hqd_pq_wptr_lo);
3582 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,
3583 mqd->cp_hqd_pq_wptr_hi);
3586 /* set the pointer to the MQD */
3587 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR,
3588 mqd->cp_mqd_base_addr_lo);
3589 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_BASE_ADDR_HI,
3590 mqd->cp_mqd_base_addr_hi);
3592 /* set MQD vmid to 0 */
3593 WREG32_SOC15_RLC(GC, 0, mmCP_MQD_CONTROL,
3594 mqd->cp_mqd_control);
3596 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
3597 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE,
3598 mqd->cp_hqd_pq_base_lo);
3599 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_BASE_HI,
3600 mqd->cp_hqd_pq_base_hi);
3602 /* set up the HQD, this is similar to CP_RB0_CNTL */
3603 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_CONTROL,
3604 mqd->cp_hqd_pq_control);
3606 /* set the wb address whether it's enabled or not */
3607 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR,
3608 mqd->cp_hqd_pq_rptr_report_addr_lo);
3609 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR_REPORT_ADDR_HI,
3610 mqd->cp_hqd_pq_rptr_report_addr_hi);
3612 /* only used if CP_PQ_WPTR_POLL_CNTL.CP_PQ_WPTR_POLL_CNTL__EN_MASK=1 */
3613 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR,
3614 mqd->cp_hqd_pq_wptr_poll_addr_lo);
3615 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_POLL_ADDR_HI,
3616 mqd->cp_hqd_pq_wptr_poll_addr_hi);
3618 /* enable the doorbell if requested */
3619 if (ring->use_doorbell) {
3620 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_LOWER,
3621 (adev->doorbell_index.kiq * 2) << 2);
3622 WREG32_SOC15(GC, 0, mmCP_MEC_DOORBELL_RANGE_UPPER,
3623 (adev->doorbell_index.userqueue_end * 2) << 2);
3626 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL,
3627 mqd->cp_hqd_pq_doorbell_control);
3629 /* reset read and write pointers, similar to CP_RB0_WPTR/_RPTR */
3630 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO,
3631 mqd->cp_hqd_pq_wptr_lo);
3632 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI,
3633 mqd->cp_hqd_pq_wptr_hi);
3635 /* set the vmid for the queue */
3636 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_VMID, mqd->cp_hqd_vmid);
3638 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE,
3639 mqd->cp_hqd_persistent_state);
3641 /* activate the queue */
3642 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE,
3643 mqd->cp_hqd_active);
3645 if (ring->use_doorbell)
3646 WREG32_FIELD15(GC, 0, CP_PQ_STATUS, DOORBELL_ENABLE, 1);
3651 static int gfx_v9_0_kiq_fini_register(struct amdgpu_ring *ring)
3653 struct amdgpu_device *adev = ring->adev;
3656 /* disable the queue if it's active */
3657 if (RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1) {
3659 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST, 1);
3661 for (j = 0; j < adev->usec_timeout; j++) {
3662 if (!(RREG32_SOC15(GC, 0, mmCP_HQD_ACTIVE) & 1))
3667 if (j == AMDGPU_MAX_USEC_TIMEOUT) {
3668 DRM_DEBUG("KIQ dequeue request failed.\n");
3670 /* Manual disable if dequeue request times out */
3671 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_ACTIVE, 0);
3674 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_DEQUEUE_REQUEST,
3678 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IQ_TIMER, 0);
3679 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_IB_CONTROL, 0);
3680 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PERSISTENT_STATE, 0);
3681 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0x40000000);
3682 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_DOORBELL_CONTROL, 0);
3683 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_RPTR, 0);
3684 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_HI, 0);
3685 WREG32_SOC15_RLC(GC, 0, mmCP_HQD_PQ_WPTR_LO, 0);
3690 static int gfx_v9_0_kiq_init_queue(struct amdgpu_ring *ring)
3692 struct amdgpu_device *adev = ring->adev;
3693 struct v9_mqd *mqd = ring->mqd_ptr;
3694 int mqd_idx = AMDGPU_MAX_COMPUTE_RINGS;
3696 gfx_v9_0_kiq_setting(ring);
3698 if (adev->in_gpu_reset) { /* for GPU_RESET case */
3699 /* reset MQD to a clean status */
3700 if (adev->gfx.mec.mqd_backup[mqd_idx])
3701 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
3703 /* reset ring buffer */
3705 amdgpu_ring_clear_ring(ring);
3707 mutex_lock(&adev->srbm_mutex);
3708 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3709 gfx_v9_0_kiq_init_register(ring);
3710 soc15_grbm_select(adev, 0, 0, 0, 0);
3711 mutex_unlock(&adev->srbm_mutex);
3713 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
3714 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
3715 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
3716 mutex_lock(&adev->srbm_mutex);
3717 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3718 gfx_v9_0_mqd_init(ring);
3719 gfx_v9_0_kiq_init_register(ring);
3720 soc15_grbm_select(adev, 0, 0, 0, 0);
3721 mutex_unlock(&adev->srbm_mutex);
3723 if (adev->gfx.mec.mqd_backup[mqd_idx])
3724 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));
3730 static int gfx_v9_0_kcq_init_queue(struct amdgpu_ring *ring)
3732 struct amdgpu_device *adev = ring->adev;
3733 struct v9_mqd *mqd = ring->mqd_ptr;
3734 int mqd_idx = ring - &adev->gfx.compute_ring[0];
3736 if (!adev->in_gpu_reset && !adev->in_suspend) {
3737 memset((void *)mqd, 0, sizeof(struct v9_mqd_allocation));
3738 ((struct v9_mqd_allocation *)mqd)->dynamic_cu_mask = 0xFFFFFFFF;
3739 ((struct v9_mqd_allocation *)mqd)->dynamic_rb_mask = 0xFFFFFFFF;
3740 mutex_lock(&adev->srbm_mutex);
3741 soc15_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0);
3742 gfx_v9_0_mqd_init(ring);
3743 soc15_grbm_select(adev, 0, 0, 0, 0);
3744 mutex_unlock(&adev->srbm_mutex);
3746 if (adev->gfx.mec.mqd_backup[mqd_idx])
3747 memcpy(adev->gfx.mec.mqd_backup[mqd_idx], mqd, sizeof(struct v9_mqd_allocation));
3748 } else if (adev->in_gpu_reset) { /* for GPU_RESET case */
3749 /* reset MQD to a clean status */
3750 if (adev->gfx.mec.mqd_backup[mqd_idx])
3751 memcpy(mqd, adev->gfx.mec.mqd_backup[mqd_idx], sizeof(struct v9_mqd_allocation));
3753 /* reset ring buffer */
3755 atomic64_set((atomic64_t *)&adev->wb.wb[ring->wptr_offs], 0);
3756 amdgpu_ring_clear_ring(ring);
3758 amdgpu_ring_clear_ring(ring);
3764 static int gfx_v9_0_kiq_resume(struct amdgpu_device *adev)
3766 struct amdgpu_ring *ring;
3769 ring = &adev->gfx.kiq.ring;
3771 r = amdgpu_bo_reserve(ring->mqd_obj, false);
3772 if (unlikely(r != 0))
3775 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
3776 if (unlikely(r != 0))
3779 gfx_v9_0_kiq_init_queue(ring);
3780 amdgpu_bo_kunmap(ring->mqd_obj);
3781 ring->mqd_ptr = NULL;
3782 amdgpu_bo_unreserve(ring->mqd_obj);
3783 ring->sched.ready = true;
3787 static int gfx_v9_0_kcq_resume(struct amdgpu_device *adev)
3789 struct amdgpu_ring *ring = NULL;
3792 gfx_v9_0_cp_compute_enable(adev, true);
3794 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3795 ring = &adev->gfx.compute_ring[i];
3797 r = amdgpu_bo_reserve(ring->mqd_obj, false);
3798 if (unlikely(r != 0))
3800 r = amdgpu_bo_kmap(ring->mqd_obj, (void **)&ring->mqd_ptr);
3802 r = gfx_v9_0_kcq_init_queue(ring);
3803 amdgpu_bo_kunmap(ring->mqd_obj);
3804 ring->mqd_ptr = NULL;
3806 amdgpu_bo_unreserve(ring->mqd_obj);
3811 r = amdgpu_gfx_enable_kcq(adev);
3816 static int gfx_v9_0_cp_resume(struct amdgpu_device *adev)
3819 struct amdgpu_ring *ring;
3821 if (!(adev->flags & AMD_IS_APU))
3822 gfx_v9_0_enable_gui_idle_interrupt(adev, false);
3824 if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
3825 if (adev->asic_type != CHIP_ARCTURUS) {
3826 /* legacy firmware loading */
3827 r = gfx_v9_0_cp_gfx_load_microcode(adev);
3832 r = gfx_v9_0_cp_compute_load_microcode(adev);
3837 r = gfx_v9_0_kiq_resume(adev);
3841 if (adev->asic_type != CHIP_ARCTURUS) {
3842 r = gfx_v9_0_cp_gfx_resume(adev);
3847 r = gfx_v9_0_kcq_resume(adev);
3851 if (adev->asic_type != CHIP_ARCTURUS) {
3852 ring = &adev->gfx.gfx_ring[0];
3853 r = amdgpu_ring_test_helper(ring);
3858 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
3859 ring = &adev->gfx.compute_ring[i];
3860 amdgpu_ring_test_helper(ring);
3863 gfx_v9_0_enable_gui_idle_interrupt(adev, true);
3868 static void gfx_v9_0_init_tcp_config(struct amdgpu_device *adev)
3872 if (adev->asic_type != CHIP_ARCTURUS)
3875 tmp = RREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG);
3876 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE64KHASH,
3877 adev->df.hash_status.hash_64k);
3878 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE2MHASH,
3879 adev->df.hash_status.hash_2m);
3880 tmp = REG_SET_FIELD(tmp, TCP_ADDR_CONFIG, ENABLE1GHASH,
3881 adev->df.hash_status.hash_1g);
3882 WREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG, tmp);
3885 static void gfx_v9_0_cp_enable(struct amdgpu_device *adev, bool enable)
3887 if (adev->asic_type != CHIP_ARCTURUS)
3888 gfx_v9_0_cp_gfx_enable(adev, enable);
3889 gfx_v9_0_cp_compute_enable(adev, enable);
3892 static int gfx_v9_0_hw_init(void *handle)
3895 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3897 if (!amdgpu_sriov_vf(adev))
3898 gfx_v9_0_init_golden_registers(adev);
3900 gfx_v9_0_constants_init(adev);
3902 gfx_v9_0_init_tcp_config(adev);
3904 r = adev->gfx.rlc.funcs->resume(adev);
3908 r = gfx_v9_0_cp_resume(adev);
3915 static int gfx_v9_0_hw_fini(void *handle)
3917 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3919 amdgpu_irq_put(adev, &adev->gfx.cp_ecc_error_irq, 0);
3920 amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
3921 amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
3923 /* DF freeze and kcq disable will fail */
3924 if (!amdgpu_ras_intr_triggered())
3925 /* disable KCQ to avoid CPC touch memory not valid anymore */
3926 amdgpu_gfx_disable_kcq(adev);
3928 if (amdgpu_sriov_vf(adev)) {
3929 gfx_v9_0_cp_gfx_enable(adev, false);
3930 /* must disable polling for SRIOV when hw finished, otherwise
3931 * CPC engine may still keep fetching WB address which is already
3932 * invalid after sw finished and trigger DMAR reading error in
3935 WREG32_FIELD15(GC, 0, CP_PQ_WPTR_POLL_CNTL, EN, 0);
3939 /* Use deinitialize sequence from CAIL when unbinding device from driver,
3940 * otherwise KIQ is hanging when binding back
3942 if (!adev->in_gpu_reset && !adev->in_suspend) {
3943 mutex_lock(&adev->srbm_mutex);
3944 soc15_grbm_select(adev, adev->gfx.kiq.ring.me,
3945 adev->gfx.kiq.ring.pipe,
3946 adev->gfx.kiq.ring.queue, 0);
3947 gfx_v9_0_kiq_fini_register(&adev->gfx.kiq.ring);
3948 soc15_grbm_select(adev, 0, 0, 0, 0);
3949 mutex_unlock(&adev->srbm_mutex);
3952 gfx_v9_0_cp_enable(adev, false);
3953 adev->gfx.rlc.funcs->stop(adev);
3958 static int gfx_v9_0_suspend(void *handle)
3960 return gfx_v9_0_hw_fini(handle);
3963 static int gfx_v9_0_resume(void *handle)
3965 return gfx_v9_0_hw_init(handle);
3968 static bool gfx_v9_0_is_idle(void *handle)
3970 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3972 if (REG_GET_FIELD(RREG32_SOC15(GC, 0, mmGRBM_STATUS),
3973 GRBM_STATUS, GUI_ACTIVE))
3979 static int gfx_v9_0_wait_for_idle(void *handle)
3982 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3984 for (i = 0; i < adev->usec_timeout; i++) {
3985 if (gfx_v9_0_is_idle(handle))
3992 static int gfx_v9_0_soft_reset(void *handle)
3994 u32 grbm_soft_reset = 0;
3996 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
3999 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS);
4000 if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK |
4001 GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK |
4002 GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK |
4003 GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK |
4004 GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK |
4005 GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK)) {
4006 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4007 GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
4008 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4009 GRBM_SOFT_RESET, SOFT_RESET_GFX, 1);
4012 if (tmp & (GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) {
4013 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4014 GRBM_SOFT_RESET, SOFT_RESET_CP, 1);
4018 tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2);
4019 if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY))
4020 grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset,
4021 GRBM_SOFT_RESET, SOFT_RESET_RLC, 1);
4024 if (grbm_soft_reset) {
4026 adev->gfx.rlc.funcs->stop(adev);
4028 if (adev->asic_type != CHIP_ARCTURUS)
4029 /* Disable GFX parsing/prefetching */
4030 gfx_v9_0_cp_gfx_enable(adev, false);
4032 /* Disable MEC parsing/prefetching */
4033 gfx_v9_0_cp_compute_enable(adev, false);
4035 if (grbm_soft_reset) {
4036 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4037 tmp |= grbm_soft_reset;
4038 dev_info(adev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4039 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
4040 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4044 tmp &= ~grbm_soft_reset;
4045 WREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET, tmp);
4046 tmp = RREG32_SOC15(GC, 0, mmGRBM_SOFT_RESET);
4049 /* Wait a little for things to settle down */
4055 static uint64_t gfx_v9_0_kiq_read_clock(struct amdgpu_device *adev)
4057 signed long r, cnt = 0;
4058 unsigned long flags;
4060 struct amdgpu_kiq *kiq = &adev->gfx.kiq;
4061 struct amdgpu_ring *ring = &kiq->ring;
4063 BUG_ON(!ring->funcs->emit_rreg);
4065 spin_lock_irqsave(&kiq->ring_lock, flags);
4066 amdgpu_ring_alloc(ring, 32);
4067 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
4068 amdgpu_ring_write(ring, 9 | /* src: register*/
4069 (5 << 8) | /* dst: memory */
4070 (1 << 16) | /* count sel */
4071 (1 << 20)); /* write confirm */
4072 amdgpu_ring_write(ring, 0);
4073 amdgpu_ring_write(ring, 0);
4074 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
4075 kiq->reg_val_offs * 4));
4076 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
4077 kiq->reg_val_offs * 4));
4078 amdgpu_fence_emit_polling(ring, &seq);
4079 amdgpu_ring_commit(ring);
4080 spin_unlock_irqrestore(&kiq->ring_lock, flags);
4082 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
4084 /* don't wait anymore for gpu reset case because this way may
4085 * block gpu_recover() routine forever, e.g. this virt_kiq_rreg
4086 * is triggered in TTM and ttm_bo_lock_delayed_workqueue() will
4087 * never return if we keep waiting in virt_kiq_rreg, which cause
4088 * gpu_recover() hang there.
4090 * also don't wait anymore for IRQ context
4092 if (r < 1 && (adev->in_gpu_reset || in_interrupt()))
4093 goto failed_kiq_read;
4096 while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) {
4097 msleep(MAX_KIQ_REG_BAILOUT_INTERVAL);
4098 r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT);
4101 if (cnt > MAX_KIQ_REG_TRY)
4102 goto failed_kiq_read;
4104 return (uint64_t)adev->wb.wb[kiq->reg_val_offs] |
4105 (uint64_t)adev->wb.wb[kiq->reg_val_offs + 1 ] << 32ULL;
4108 pr_err("failed to read gpu clock\n");
4112 static uint64_t gfx_v9_0_get_gpu_clock_counter(struct amdgpu_device *adev)
4116 amdgpu_gfx_off_ctrl(adev, false);
4117 mutex_lock(&adev->gfx.gpu_clock_mutex);
4118 if (adev->asic_type == CHIP_VEGA10 && amdgpu_sriov_runtime(adev)) {
4119 clock = gfx_v9_0_kiq_read_clock(adev);
4121 WREG32_SOC15(GC, 0, mmRLC_CAPTURE_GPU_CLOCK_COUNT, 1);
4122 clock = (uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_LSB) |
4123 ((uint64_t)RREG32_SOC15(GC, 0, mmRLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
4125 mutex_unlock(&adev->gfx.gpu_clock_mutex);
4126 amdgpu_gfx_off_ctrl(adev, true);
4130 static void gfx_v9_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
4132 uint32_t gds_base, uint32_t gds_size,
4133 uint32_t gws_base, uint32_t gws_size,
4134 uint32_t oa_base, uint32_t oa_size)
4136 struct amdgpu_device *adev = ring->adev;
4139 gfx_v9_0_write_data_to_reg(ring, 0, false,
4140 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_BASE) + 2 * vmid,
4144 gfx_v9_0_write_data_to_reg(ring, 0, false,
4145 SOC15_REG_OFFSET(GC, 0, mmGDS_VMID0_SIZE) + 2 * vmid,
4149 gfx_v9_0_write_data_to_reg(ring, 0, false,
4150 SOC15_REG_OFFSET(GC, 0, mmGDS_GWS_VMID0) + vmid,
4151 gws_size << GDS_GWS_VMID0__SIZE__SHIFT | gws_base);
4154 gfx_v9_0_write_data_to_reg(ring, 0, false,
4155 SOC15_REG_OFFSET(GC, 0, mmGDS_OA_VMID0) + vmid,
4156 (1 << (oa_size + oa_base)) - (1 << oa_base));
4159 static const u32 vgpr_init_compute_shader[] =
4161 0xb07c0000, 0xbe8000ff,
4162 0x000000f8, 0xbf110800,
4163 0x7e000280, 0x7e020280,
4164 0x7e040280, 0x7e060280,
4165 0x7e080280, 0x7e0a0280,
4166 0x7e0c0280, 0x7e0e0280,
4167 0x80808800, 0xbe803200,
4168 0xbf84fff5, 0xbf9c0000,
4169 0xd28c0001, 0x0001007f,
4170 0xd28d0001, 0x0002027e,
4171 0x10020288, 0xb8810904,
4172 0xb7814000, 0xd1196a01,
4173 0x00000301, 0xbe800087,
4174 0xbefc00c1, 0xd89c4000,
4175 0x00020201, 0xd89cc080,
4176 0x00040401, 0x320202ff,
4177 0x00000800, 0x80808100,
4178 0xbf84fff8, 0x7e020280,
4179 0xbf810000, 0x00000000,
4182 static const u32 sgpr_init_compute_shader[] =
4184 0xb07c0000, 0xbe8000ff,
4185 0x0000005f, 0xbee50080,
4186 0xbe812c65, 0xbe822c65,
4187 0xbe832c65, 0xbe842c65,
4188 0xbe852c65, 0xb77c0005,
4189 0x80808500, 0xbf84fff8,
4190 0xbe800080, 0xbf810000,
4193 static const u32 vgpr_init_compute_shader_arcturus[] = {
4194 0xd3d94000, 0x18000080, 0xd3d94001, 0x18000080, 0xd3d94002, 0x18000080,
4195 0xd3d94003, 0x18000080, 0xd3d94004, 0x18000080, 0xd3d94005, 0x18000080,
4196 0xd3d94006, 0x18000080, 0xd3d94007, 0x18000080, 0xd3d94008, 0x18000080,
4197 0xd3d94009, 0x18000080, 0xd3d9400a, 0x18000080, 0xd3d9400b, 0x18000080,
4198 0xd3d9400c, 0x18000080, 0xd3d9400d, 0x18000080, 0xd3d9400e, 0x18000080,
4199 0xd3d9400f, 0x18000080, 0xd3d94010, 0x18000080, 0xd3d94011, 0x18000080,
4200 0xd3d94012, 0x18000080, 0xd3d94013, 0x18000080, 0xd3d94014, 0x18000080,
4201 0xd3d94015, 0x18000080, 0xd3d94016, 0x18000080, 0xd3d94017, 0x18000080,
4202 0xd3d94018, 0x18000080, 0xd3d94019, 0x18000080, 0xd3d9401a, 0x18000080,
4203 0xd3d9401b, 0x18000080, 0xd3d9401c, 0x18000080, 0xd3d9401d, 0x18000080,
4204 0xd3d9401e, 0x18000080, 0xd3d9401f, 0x18000080, 0xd3d94020, 0x18000080,
4205 0xd3d94021, 0x18000080, 0xd3d94022, 0x18000080, 0xd3d94023, 0x18000080,
4206 0xd3d94024, 0x18000080, 0xd3d94025, 0x18000080, 0xd3d94026, 0x18000080,
4207 0xd3d94027, 0x18000080, 0xd3d94028, 0x18000080, 0xd3d94029, 0x18000080,
4208 0xd3d9402a, 0x18000080, 0xd3d9402b, 0x18000080, 0xd3d9402c, 0x18000080,
4209 0xd3d9402d, 0x18000080, 0xd3d9402e, 0x18000080, 0xd3d9402f, 0x18000080,
4210 0xd3d94030, 0x18000080, 0xd3d94031, 0x18000080, 0xd3d94032, 0x18000080,
4211 0xd3d94033, 0x18000080, 0xd3d94034, 0x18000080, 0xd3d94035, 0x18000080,
4212 0xd3d94036, 0x18000080, 0xd3d94037, 0x18000080, 0xd3d94038, 0x18000080,
4213 0xd3d94039, 0x18000080, 0xd3d9403a, 0x18000080, 0xd3d9403b, 0x18000080,
4214 0xd3d9403c, 0x18000080, 0xd3d9403d, 0x18000080, 0xd3d9403e, 0x18000080,
4215 0xd3d9403f, 0x18000080, 0xd3d94040, 0x18000080, 0xd3d94041, 0x18000080,
4216 0xd3d94042, 0x18000080, 0xd3d94043, 0x18000080, 0xd3d94044, 0x18000080,
4217 0xd3d94045, 0x18000080, 0xd3d94046, 0x18000080, 0xd3d94047, 0x18000080,
4218 0xd3d94048, 0x18000080, 0xd3d94049, 0x18000080, 0xd3d9404a, 0x18000080,
4219 0xd3d9404b, 0x18000080, 0xd3d9404c, 0x18000080, 0xd3d9404d, 0x18000080,
4220 0xd3d9404e, 0x18000080, 0xd3d9404f, 0x18000080, 0xd3d94050, 0x18000080,
4221 0xd3d94051, 0x18000080, 0xd3d94052, 0x18000080, 0xd3d94053, 0x18000080,
4222 0xd3d94054, 0x18000080, 0xd3d94055, 0x18000080, 0xd3d94056, 0x18000080,
4223 0xd3d94057, 0x18000080, 0xd3d94058, 0x18000080, 0xd3d94059, 0x18000080,
4224 0xd3d9405a, 0x18000080, 0xd3d9405b, 0x18000080, 0xd3d9405c, 0x18000080,
4225 0xd3d9405d, 0x18000080, 0xd3d9405e, 0x18000080, 0xd3d9405f, 0x18000080,
4226 0xd3d94060, 0x18000080, 0xd3d94061, 0x18000080, 0xd3d94062, 0x18000080,
4227 0xd3d94063, 0x18000080, 0xd3d94064, 0x18000080, 0xd3d94065, 0x18000080,
4228 0xd3d94066, 0x18000080, 0xd3d94067, 0x18000080, 0xd3d94068, 0x18000080,
4229 0xd3d94069, 0x18000080, 0xd3d9406a, 0x18000080, 0xd3d9406b, 0x18000080,
4230 0xd3d9406c, 0x18000080, 0xd3d9406d, 0x18000080, 0xd3d9406e, 0x18000080,
4231 0xd3d9406f, 0x18000080, 0xd3d94070, 0x18000080, 0xd3d94071, 0x18000080,
4232 0xd3d94072, 0x18000080, 0xd3d94073, 0x18000080, 0xd3d94074, 0x18000080,
4233 0xd3d94075, 0x18000080, 0xd3d94076, 0x18000080, 0xd3d94077, 0x18000080,
4234 0xd3d94078, 0x18000080, 0xd3d94079, 0x18000080, 0xd3d9407a, 0x18000080,
4235 0xd3d9407b, 0x18000080, 0xd3d9407c, 0x18000080, 0xd3d9407d, 0x18000080,
4236 0xd3d9407e, 0x18000080, 0xd3d9407f, 0x18000080, 0xd3d94080, 0x18000080,
4237 0xd3d94081, 0x18000080, 0xd3d94082, 0x18000080, 0xd3d94083, 0x18000080,
4238 0xd3d94084, 0x18000080, 0xd3d94085, 0x18000080, 0xd3d94086, 0x18000080,
4239 0xd3d94087, 0x18000080, 0xd3d94088, 0x18000080, 0xd3d94089, 0x18000080,
4240 0xd3d9408a, 0x18000080, 0xd3d9408b, 0x18000080, 0xd3d9408c, 0x18000080,
4241 0xd3d9408d, 0x18000080, 0xd3d9408e, 0x18000080, 0xd3d9408f, 0x18000080,
4242 0xd3d94090, 0x18000080, 0xd3d94091, 0x18000080, 0xd3d94092, 0x18000080,
4243 0xd3d94093, 0x18000080, 0xd3d94094, 0x18000080, 0xd3d94095, 0x18000080,
4244 0xd3d94096, 0x18000080, 0xd3d94097, 0x18000080, 0xd3d94098, 0x18000080,
4245 0xd3d94099, 0x18000080, 0xd3d9409a, 0x18000080, 0xd3d9409b, 0x18000080,
4246 0xd3d9409c, 0x18000080, 0xd3d9409d, 0x18000080, 0xd3d9409e, 0x18000080,
4247 0xd3d9409f, 0x18000080, 0xd3d940a0, 0x18000080, 0xd3d940a1, 0x18000080,
4248 0xd3d940a2, 0x18000080, 0xd3d940a3, 0x18000080, 0xd3d940a4, 0x18000080,
4249 0xd3d940a5, 0x18000080, 0xd3d940a6, 0x18000080, 0xd3d940a7, 0x18000080,
4250 0xd3d940a8, 0x18000080, 0xd3d940a9, 0x18000080, 0xd3d940aa, 0x18000080,
4251 0xd3d940ab, 0x18000080, 0xd3d940ac, 0x18000080, 0xd3d940ad, 0x18000080,
4252 0xd3d940ae, 0x18000080, 0xd3d940af, 0x18000080, 0xd3d940b0, 0x18000080,
4253 0xd3d940b1, 0x18000080, 0xd3d940b2, 0x18000080, 0xd3d940b3, 0x18000080,
4254 0xd3d940b4, 0x18000080, 0xd3d940b5, 0x18000080, 0xd3d940b6, 0x18000080,
4255 0xd3d940b7, 0x18000080, 0xd3d940b8, 0x18000080, 0xd3d940b9, 0x18000080,
4256 0xd3d940ba, 0x18000080, 0xd3d940bb, 0x18000080, 0xd3d940bc, 0x18000080,
4257 0xd3d940bd, 0x18000080, 0xd3d940be, 0x18000080, 0xd3d940bf, 0x18000080,
4258 0xd3d940c0, 0x18000080, 0xd3d940c1, 0x18000080, 0xd3d940c2, 0x18000080,
4259 0xd3d940c3, 0x18000080, 0xd3d940c4, 0x18000080, 0xd3d940c5, 0x18000080,
4260 0xd3d940c6, 0x18000080, 0xd3d940c7, 0x18000080, 0xd3d940c8, 0x18000080,
4261 0xd3d940c9, 0x18000080, 0xd3d940ca, 0x18000080, 0xd3d940cb, 0x18000080,
4262 0xd3d940cc, 0x18000080, 0xd3d940cd, 0x18000080, 0xd3d940ce, 0x18000080,
4263 0xd3d940cf, 0x18000080, 0xd3d940d0, 0x18000080, 0xd3d940d1, 0x18000080,
4264 0xd3d940d2, 0x18000080, 0xd3d940d3, 0x18000080, 0xd3d940d4, 0x18000080,
4265 0xd3d940d5, 0x18000080, 0xd3d940d6, 0x18000080, 0xd3d940d7, 0x18000080,
4266 0xd3d940d8, 0x18000080, 0xd3d940d9, 0x18000080, 0xd3d940da, 0x18000080,
4267 0xd3d940db, 0x18000080, 0xd3d940dc, 0x18000080, 0xd3d940dd, 0x18000080,
4268 0xd3d940de, 0x18000080, 0xd3d940df, 0x18000080, 0xd3d940e0, 0x18000080,
4269 0xd3d940e1, 0x18000080, 0xd3d940e2, 0x18000080, 0xd3d940e3, 0x18000080,
4270 0xd3d940e4, 0x18000080, 0xd3d940e5, 0x18000080, 0xd3d940e6, 0x18000080,
4271 0xd3d940e7, 0x18000080, 0xd3d940e8, 0x18000080, 0xd3d940e9, 0x18000080,
4272 0xd3d940ea, 0x18000080, 0xd3d940eb, 0x18000080, 0xd3d940ec, 0x18000080,
4273 0xd3d940ed, 0x18000080, 0xd3d940ee, 0x18000080, 0xd3d940ef, 0x18000080,
4274 0xd3d940f0, 0x18000080, 0xd3d940f1, 0x18000080, 0xd3d940f2, 0x18000080,
4275 0xd3d940f3, 0x18000080, 0xd3d940f4, 0x18000080, 0xd3d940f5, 0x18000080,
4276 0xd3d940f6, 0x18000080, 0xd3d940f7, 0x18000080, 0xd3d940f8, 0x18000080,
4277 0xd3d940f9, 0x18000080, 0xd3d940fa, 0x18000080, 0xd3d940fb, 0x18000080,
4278 0xd3d940fc, 0x18000080, 0xd3d940fd, 0x18000080, 0xd3d940fe, 0x18000080,
4279 0xd3d940ff, 0x18000080, 0xb07c0000, 0xbe8a00ff, 0x000000f8, 0xbf11080a,
4280 0x7e000280, 0x7e020280, 0x7e040280, 0x7e060280, 0x7e080280, 0x7e0a0280,
4281 0x7e0c0280, 0x7e0e0280, 0x808a880a, 0xbe80320a, 0xbf84fff5, 0xbf9c0000,
4282 0xd28c0001, 0x0001007f, 0xd28d0001, 0x0002027e, 0x10020288, 0xb88b0904,
4283 0xb78b4000, 0xd1196a01, 0x00001701, 0xbe8a0087, 0xbefc00c1, 0xd89c4000,
4284 0x00020201, 0xd89cc080, 0x00040401, 0x320202ff, 0x00000800, 0x808a810a,
4285 0xbf84fff8, 0xbf810000,
4288 /* When below register arrays changed, please update gpr_reg_size,
4289 and sec_ded_counter_reg_size in function gfx_v9_0_do_edc_gpr_workarounds,
4290 to cover all gfx9 ASICs */
4291 static const struct soc15_reg_entry vgpr_init_regs[] = {
4292 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4293 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4294 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },
4295 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4296 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x3f },
4297 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */
4298 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
4299 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
4300 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
4301 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
4302 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
4303 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
4304 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
4305 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
4308 static const struct soc15_reg_entry vgpr_init_regs_arcturus[] = {
4309 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4310 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4311 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 4 },
4312 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4313 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0xbf },
4314 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x400000 }, /* 64KB LDS */
4315 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
4316 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
4317 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
4318 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
4319 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
4320 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
4321 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
4322 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
4325 static const struct soc15_reg_entry sgpr1_init_regs[] = {
4326 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4327 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4328 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 },
4329 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4330 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */
4331 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },
4332 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x000000ff },
4333 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x000000ff },
4334 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x000000ff },
4335 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x000000ff },
4336 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x000000ff },
4337 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x000000ff },
4338 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x000000ff },
4339 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x000000ff },
4342 static const struct soc15_reg_entry sgpr2_init_regs[] = {
4343 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
4344 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_X), 0x40 },
4345 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Y), 8 },
4346 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_NUM_THREAD_Z), 1 },
4347 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC1), 0x240 }, /* (80 GPRS) */
4348 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_PGM_RSRC2), 0x0 },
4349 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE0), 0x0000ff00 },
4350 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE1), 0x0000ff00 },
4351 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE2), 0x0000ff00 },
4352 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE3), 0x0000ff00 },
4353 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE4), 0x0000ff00 },
4354 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE5), 0x0000ff00 },
4355 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE6), 0x0000ff00 },
4356 { SOC15_REG_ENTRY(GC, 0, mmCOMPUTE_STATIC_THREAD_MGMT_SE7), 0x0000ff00 },
4359 static const struct soc15_reg_entry gfx_v9_0_edc_counter_regs[] = {
4360 { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT), 0, 1, 1},
4361 { SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT), 0, 1, 1},
4362 { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT), 0, 1, 1},
4363 { SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT), 0, 1, 1},
4364 { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT), 0, 1, 1},
4365 { SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT), 0, 1, 1},
4366 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT), 0, 1, 1},
4367 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT), 0, 1, 1},
4368 { SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT), 0, 1, 1},
4369 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT), 0, 1, 1},
4370 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_GRBM_CNT), 0, 1, 1},
4371 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_DED), 0, 1, 1},
4372 { SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT), 0, 4, 1},
4373 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT), 0, 4, 6},
4374 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_DED_CNT), 0, 4, 16},
4375 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_INFO), 0, 4, 16},
4376 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_SEC_CNT), 0, 4, 16},
4377 { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT), 0, 1, 16},
4378 { SOC15_REG_ENTRY(GC, 0, mmTCP_ATC_EDC_GATCL1_CNT), 0, 4, 16},
4379 { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT), 0, 4, 16},
4380 { SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW), 0, 4, 16},
4381 { SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT), 0, 4, 16},
4382 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2), 0, 4, 6},
4383 { SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT), 0, 4, 16},
4384 { SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT), 0, 4, 16},
4385 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT), 0, 1, 1},
4386 { SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT), 0, 1, 1},
4387 { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT), 0, 1, 32},
4388 { SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2), 0, 1, 32},
4389 { SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT), 0, 1, 72},
4390 { SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2), 0, 1, 16},
4391 { SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT), 0, 1, 2},
4392 { SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3), 0, 4, 6},
4395 static int gfx_v9_0_do_edc_gds_workarounds(struct amdgpu_device *adev)
4397 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
4400 /* only support when RAS is enabled */
4401 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
4404 r = amdgpu_ring_alloc(ring, 7);
4406 DRM_ERROR("amdgpu: GDS workarounds failed to lock ring %s (%d).\n",
4411 WREG32_SOC15(GC, 0, mmGDS_VMID0_BASE, 0x00000000);
4412 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, adev->gds.gds_size);
4414 amdgpu_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
4415 amdgpu_ring_write(ring, (PACKET3_DMA_DATA_CP_SYNC |
4416 PACKET3_DMA_DATA_DST_SEL(1) |
4417 PACKET3_DMA_DATA_SRC_SEL(2) |
4418 PACKET3_DMA_DATA_ENGINE(0)));
4419 amdgpu_ring_write(ring, 0);
4420 amdgpu_ring_write(ring, 0);
4421 amdgpu_ring_write(ring, 0);
4422 amdgpu_ring_write(ring, 0);
4423 amdgpu_ring_write(ring, PACKET3_DMA_DATA_CMD_RAW_WAIT |
4424 adev->gds.gds_size);
4426 amdgpu_ring_commit(ring);
4428 for (i = 0; i < adev->usec_timeout; i++) {
4429 if (ring->wptr == gfx_v9_0_ring_get_rptr_compute(ring))
4434 if (i >= adev->usec_timeout)
4437 WREG32_SOC15(GC, 0, mmGDS_VMID0_SIZE, 0x00000000);
4442 static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)
4444 struct amdgpu_ring *ring = &adev->gfx.compute_ring[0];
4445 struct amdgpu_ib ib;
4446 struct dma_fence *f = NULL;
4448 unsigned total_size, vgpr_offset, sgpr_offset;
4451 int compute_dim_x = adev->gfx.config.max_shader_engines *
4452 adev->gfx.config.max_cu_per_sh *
4453 adev->gfx.config.max_sh_per_se;
4454 int sgpr_work_group_size = 5;
4455 int gpr_reg_size = adev->gfx.config.max_shader_engines + 6;
4456 int vgpr_init_shader_size;
4457 const u32 *vgpr_init_shader_ptr;
4458 const struct soc15_reg_entry *vgpr_init_regs_ptr;
4460 /* only support when RAS is enabled */
4461 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
4464 /* bail if the compute ring is not ready */
4465 if (!ring->sched.ready)
4468 if (adev->asic_type == CHIP_ARCTURUS) {
4469 vgpr_init_shader_ptr = vgpr_init_compute_shader_arcturus;
4470 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader_arcturus);
4471 vgpr_init_regs_ptr = vgpr_init_regs_arcturus;
4473 vgpr_init_shader_ptr = vgpr_init_compute_shader;
4474 vgpr_init_shader_size = sizeof(vgpr_init_compute_shader);
4475 vgpr_init_regs_ptr = vgpr_init_regs;
4479 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* VGPRS */
4481 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS1 */
4483 (gpr_reg_size * 3 + 4 + 5 + 2) * 4; /* SGPRS2 */
4484 total_size = ALIGN(total_size, 256);
4485 vgpr_offset = total_size;
4486 total_size += ALIGN(vgpr_init_shader_size, 256);
4487 sgpr_offset = total_size;
4488 total_size += sizeof(sgpr_init_compute_shader);
4490 /* allocate an indirect buffer to put the commands in */
4491 memset(&ib, 0, sizeof(ib));
4492 r = amdgpu_ib_get(adev, NULL, total_size, &ib);
4494 DRM_ERROR("amdgpu: failed to get ib (%d).\n", r);
4498 /* load the compute shaders */
4499 for (i = 0; i < vgpr_init_shader_size/sizeof(u32); i++)
4500 ib.ptr[i + (vgpr_offset / 4)] = vgpr_init_shader_ptr[i];
4502 for (i = 0; i < ARRAY_SIZE(sgpr_init_compute_shader); i++)
4503 ib.ptr[i + (sgpr_offset / 4)] = sgpr_init_compute_shader[i];
4505 /* init the ib length to 0 */
4509 /* write the register state for the compute dispatch */
4510 for (i = 0; i < gpr_reg_size; i++) {
4511 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4512 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(vgpr_init_regs_ptr[i])
4513 - PACKET3_SET_SH_REG_START;
4514 ib.ptr[ib.length_dw++] = vgpr_init_regs_ptr[i].reg_value;
4516 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4517 gpu_addr = (ib.gpu_addr + (u64)vgpr_offset) >> 8;
4518 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4519 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4520 - PACKET3_SET_SH_REG_START;
4521 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4522 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4524 /* write dispatch packet */
4525 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4526 ib.ptr[ib.length_dw++] = compute_dim_x * 2; /* x */
4527 ib.ptr[ib.length_dw++] = 1; /* y */
4528 ib.ptr[ib.length_dw++] = 1; /* z */
4529 ib.ptr[ib.length_dw++] =
4530 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4532 /* write CS partial flush packet */
4533 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4534 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4537 /* write the register state for the compute dispatch */
4538 for (i = 0; i < gpr_reg_size; i++) {
4539 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4540 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr1_init_regs[i])
4541 - PACKET3_SET_SH_REG_START;
4542 ib.ptr[ib.length_dw++] = sgpr1_init_regs[i].reg_value;
4544 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4545 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
4546 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4547 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4548 - PACKET3_SET_SH_REG_START;
4549 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4550 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4552 /* write dispatch packet */
4553 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4554 ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */
4555 ib.ptr[ib.length_dw++] = 1; /* y */
4556 ib.ptr[ib.length_dw++] = 1; /* z */
4557 ib.ptr[ib.length_dw++] =
4558 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4560 /* write CS partial flush packet */
4561 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4562 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4565 /* write the register state for the compute dispatch */
4566 for (i = 0; i < gpr_reg_size; i++) {
4567 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
4568 ib.ptr[ib.length_dw++] = SOC15_REG_ENTRY_OFFSET(sgpr2_init_regs[i])
4569 - PACKET3_SET_SH_REG_START;
4570 ib.ptr[ib.length_dw++] = sgpr2_init_regs[i].reg_value;
4572 /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
4573 gpu_addr = (ib.gpu_addr + (u64)sgpr_offset) >> 8;
4574 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
4575 ib.ptr[ib.length_dw++] = SOC15_REG_OFFSET(GC, 0, mmCOMPUTE_PGM_LO)
4576 - PACKET3_SET_SH_REG_START;
4577 ib.ptr[ib.length_dw++] = lower_32_bits(gpu_addr);
4578 ib.ptr[ib.length_dw++] = upper_32_bits(gpu_addr);
4580 /* write dispatch packet */
4581 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
4582 ib.ptr[ib.length_dw++] = compute_dim_x / 2 * sgpr_work_group_size; /* x */
4583 ib.ptr[ib.length_dw++] = 1; /* y */
4584 ib.ptr[ib.length_dw++] = 1; /* z */
4585 ib.ptr[ib.length_dw++] =
4586 REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
4588 /* write CS partial flush packet */
4589 ib.ptr[ib.length_dw++] = PACKET3(PACKET3_EVENT_WRITE, 0);
4590 ib.ptr[ib.length_dw++] = EVENT_TYPE(7) | EVENT_INDEX(4);
4592 /* shedule the ib on the ring */
4593 r = amdgpu_ib_schedule(ring, 1, &ib, NULL, &f);
4595 DRM_ERROR("amdgpu: ib submit failed (%d).\n", r);
4599 /* wait for the GPU to finish processing the IB */
4600 r = dma_fence_wait(f, false);
4602 DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
4607 amdgpu_ib_free(adev, &ib, NULL);
4613 static int gfx_v9_0_early_init(void *handle)
4615 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4617 if (adev->asic_type == CHIP_ARCTURUS)
4618 adev->gfx.num_gfx_rings = 0;
4620 adev->gfx.num_gfx_rings = GFX9_NUM_GFX_RINGS;
4621 adev->gfx.num_compute_rings = AMDGPU_MAX_COMPUTE_RINGS;
4622 gfx_v9_0_set_kiq_pm4_funcs(adev);
4623 gfx_v9_0_set_ring_funcs(adev);
4624 gfx_v9_0_set_irq_funcs(adev);
4625 gfx_v9_0_set_gds_init(adev);
4626 gfx_v9_0_set_rlc_funcs(adev);
4631 static int gfx_v9_0_ecc_late_init(void *handle)
4633 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4637 * Temp workaround to fix the issue that CP firmware fails to
4638 * update read pointer when CPDMA is writing clearing operation
4639 * to GDS in suspend/resume sequence on several cards. So just
4640 * limit this operation in cold boot sequence.
4642 if (!adev->in_suspend) {
4643 r = gfx_v9_0_do_edc_gds_workarounds(adev);
4648 /* requires IBs so do in late init after IB pool is initialized */
4649 r = gfx_v9_0_do_edc_gpr_workarounds(adev);
4653 if (adev->gfx.funcs &&
4654 adev->gfx.funcs->reset_ras_error_count)
4655 adev->gfx.funcs->reset_ras_error_count(adev);
4657 r = amdgpu_gfx_ras_late_init(adev);
4664 static int gfx_v9_0_late_init(void *handle)
4666 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
4669 r = amdgpu_irq_get(adev, &adev->gfx.priv_reg_irq, 0);
4673 r = amdgpu_irq_get(adev, &adev->gfx.priv_inst_irq, 0);
4677 r = gfx_v9_0_ecc_late_init(handle);
4684 static bool gfx_v9_0_is_rlc_enabled(struct amdgpu_device *adev)
4686 uint32_t rlc_setting;
4688 /* if RLC is not enabled, do nothing */
4689 rlc_setting = RREG32_SOC15(GC, 0, mmRLC_CNTL);
4690 if (!(rlc_setting & RLC_CNTL__RLC_ENABLE_F32_MASK))
4696 static void gfx_v9_0_set_safe_mode(struct amdgpu_device *adev)
4701 data = RLC_SAFE_MODE__CMD_MASK;
4702 data |= (1 << RLC_SAFE_MODE__MESSAGE__SHIFT);
4703 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
4705 /* wait for RLC_SAFE_MODE */
4706 for (i = 0; i < adev->usec_timeout; i++) {
4707 if (!REG_GET_FIELD(RREG32_SOC15(GC, 0, mmRLC_SAFE_MODE), RLC_SAFE_MODE, CMD))
4713 static void gfx_v9_0_unset_safe_mode(struct amdgpu_device *adev)
4717 data = RLC_SAFE_MODE__CMD_MASK;
4718 WREG32_SOC15(GC, 0, mmRLC_SAFE_MODE, data);
4721 static void gfx_v9_0_update_gfx_cg_power_gating(struct amdgpu_device *adev,
4724 amdgpu_gfx_rlc_enter_safe_mode(adev);
4726 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) {
4727 gfx_v9_0_enable_gfx_cg_power_gating(adev, true);
4728 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
4729 gfx_v9_0_enable_gfx_pipeline_powergating(adev, true);
4731 gfx_v9_0_enable_gfx_cg_power_gating(adev, false);
4732 if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
4733 gfx_v9_0_enable_gfx_pipeline_powergating(adev, false);
4736 amdgpu_gfx_rlc_exit_safe_mode(adev);
4739 static void gfx_v9_0_update_gfx_mg_power_gating(struct amdgpu_device *adev,
4742 /* TODO: double check if we need to perform under safe mode */
4743 /* gfx_v9_0_enter_rlc_safe_mode(adev); */
4745 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable)
4746 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, true);
4748 gfx_v9_0_enable_gfx_static_mg_power_gating(adev, false);
4750 if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable)
4751 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, true);
4753 gfx_v9_0_enable_gfx_dynamic_mg_power_gating(adev, false);
4755 /* gfx_v9_0_exit_rlc_safe_mode(adev); */
4758 static void gfx_v9_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
4763 amdgpu_gfx_rlc_enter_safe_mode(adev);
4765 /* It is disabled by HW by default */
4766 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGCG)) {
4767 /* 1 - RLC_CGTT_MGCG_OVERRIDE */
4768 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4770 if (adev->asic_type != CHIP_VEGA12)
4771 data &= ~RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK;
4773 data &= ~(RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
4774 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
4775 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
4777 /* only for Vega10 & Raven1 */
4778 data |= RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK;
4781 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4783 /* MGLS is a global flag to control all MGLS in GFX */
4784 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_MGLS) {
4785 /* 2 - RLC memory Light sleep */
4786 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_RLC_LS) {
4787 def = data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
4788 data |= RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
4790 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
4792 /* 3 - CP memory Light sleep */
4793 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CP_LS) {
4794 def = data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
4795 data |= CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
4797 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
4801 /* 1 - MGCG_OVERRIDE */
4802 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4804 if (adev->asic_type != CHIP_VEGA12)
4805 data |= RLC_CGTT_MGCG_OVERRIDE__CPF_CGTT_SCLK_OVERRIDE_MASK;
4807 data |= (RLC_CGTT_MGCG_OVERRIDE__RLC_CGTT_SCLK_OVERRIDE_MASK |
4808 RLC_CGTT_MGCG_OVERRIDE__GRBM_CGTT_SCLK_OVERRIDE_MASK |
4809 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK |
4810 RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGLS_OVERRIDE_MASK);
4813 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4815 /* 2 - disable MGLS in RLC */
4816 data = RREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL);
4817 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK) {
4818 data &= ~RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK;
4819 WREG32_SOC15(GC, 0, mmRLC_MEM_SLP_CNTL, data);
4822 /* 3 - disable MGLS in CP */
4823 data = RREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL);
4824 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK) {
4825 data &= ~CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK;
4826 WREG32_SOC15(GC, 0, mmCP_MEM_SLP_CNTL, data);
4830 amdgpu_gfx_rlc_exit_safe_mode(adev);
4833 static void gfx_v9_0_update_3d_clock_gating(struct amdgpu_device *adev,
4838 if (adev->asic_type == CHIP_ARCTURUS)
4841 amdgpu_gfx_rlc_enter_safe_mode(adev);
4843 /* Enable 3D CGCG/CGLS */
4844 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGCG)) {
4845 /* write cmd to clear cgcg/cgls ov */
4846 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4847 /* unset CGCG override */
4848 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_GFX3D_CG_OVERRIDE_MASK;
4849 /* update CGCG and CGLS override bits */
4851 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4853 /* enable 3Dcgcg FSM(0x0000363f) */
4854 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
4856 data = (0x36 << RLC_CGCG_CGLS_CTRL_3D__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
4857 RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK;
4858 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_3D_CGLS)
4859 data |= (0x000F << RLC_CGCG_CGLS_CTRL_3D__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
4860 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK;
4862 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
4864 /* set IDLE_POLL_COUNT(0x00900100) */
4865 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
4866 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
4867 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
4869 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
4871 /* Disable CGCG/CGLS */
4872 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D);
4873 /* disable cgcg, cgls should be disabled */
4874 data &= ~(RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK |
4875 RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK);
4876 /* disable cgcg and cgls in FSM */
4878 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D, data);
4881 amdgpu_gfx_rlc_exit_safe_mode(adev);
4884 static void gfx_v9_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev,
4889 amdgpu_gfx_rlc_enter_safe_mode(adev);
4891 if (enable && (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGCG)) {
4892 def = data = RREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE);
4893 /* unset CGCG override */
4894 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGCG_OVERRIDE_MASK;
4895 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
4896 data &= ~RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
4898 data |= RLC_CGTT_MGCG_OVERRIDE__GFXIP_CGLS_OVERRIDE_MASK;
4899 /* update CGCG and CGLS override bits */
4901 WREG32_SOC15(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE, data);
4903 /* enable cgcg FSM(0x0000363F) */
4904 def = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
4906 if (adev->asic_type == CHIP_ARCTURUS)
4907 data = (0x2000 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
4908 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
4910 data = (0x36 << RLC_CGCG_CGLS_CTRL__CGCG_GFX_IDLE_THRESHOLD__SHIFT) |
4911 RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK;
4912 if (adev->cg_flags & AMD_CG_SUPPORT_GFX_CGLS)
4913 data |= (0x000F << RLC_CGCG_CGLS_CTRL__CGLS_REP_COMPANSAT_DELAY__SHIFT) |
4914 RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK;
4916 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
4918 /* set IDLE_POLL_COUNT(0x00900100) */
4919 def = RREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL);
4920 data = (0x0100 << CP_RB_WPTR_POLL_CNTL__POLL_FREQUENCY__SHIFT) |
4921 (0x0090 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
4923 WREG32_SOC15(GC, 0, mmCP_RB_WPTR_POLL_CNTL, data);
4925 def = data = RREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL);
4926 /* reset CGCG/CGLS bits */
4927 data &= ~(RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK | RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK);
4928 /* disable cgcg and cgls in FSM */
4930 WREG32_SOC15(GC, 0, mmRLC_CGCG_CGLS_CTRL, data);
4933 amdgpu_gfx_rlc_exit_safe_mode(adev);
4936 static int gfx_v9_0_update_gfx_clock_gating(struct amdgpu_device *adev,
4940 /* CGCG/CGLS should be enabled after MGCG/MGLS
4941 * === MGCG + MGLS ===
4943 gfx_v9_0_update_medium_grain_clock_gating(adev, enable);
4944 /* === CGCG /CGLS for GFX 3D Only === */
4945 gfx_v9_0_update_3d_clock_gating(adev, enable);
4946 /* === CGCG + CGLS === */
4947 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);
4949 /* CGCG/CGLS should be disabled before MGCG/MGLS
4950 * === CGCG + CGLS ===
4952 gfx_v9_0_update_coarse_grain_clock_gating(adev, enable);
4953 /* === CGCG /CGLS for GFX 3D Only === */
4954 gfx_v9_0_update_3d_clock_gating(adev, enable);
4955 /* === MGCG + MGLS === */
4956 gfx_v9_0_update_medium_grain_clock_gating(adev, enable);
4961 static void gfx_v9_0_update_spm_vmid(struct amdgpu_device *adev, unsigned vmid)
4965 data = RREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL);
4967 data &= ~RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK;
4968 data |= (vmid & RLC_SPM_MC_CNTL__RLC_SPM_VMID_MASK) << RLC_SPM_MC_CNTL__RLC_SPM_VMID__SHIFT;
4970 WREG32_SOC15(GC, 0, mmRLC_SPM_MC_CNTL, data);
4973 static bool gfx_v9_0_check_rlcg_range(struct amdgpu_device *adev,
4975 struct soc15_reg_rlcg *entries, int arr_size)
4983 for (i = 0; i < arr_size; i++) {
4984 const struct soc15_reg_rlcg *entry;
4986 entry = &entries[i];
4987 reg = adev->reg_offset[entry->hwip][entry->instance][entry->segment] + entry->reg;
4995 static bool gfx_v9_0_is_rlcg_access_range(struct amdgpu_device *adev, u32 offset)
4997 return gfx_v9_0_check_rlcg_range(adev, offset,
4998 (void *)rlcg_access_gc_9_0,
4999 ARRAY_SIZE(rlcg_access_gc_9_0));
5002 static const struct amdgpu_rlc_funcs gfx_v9_0_rlc_funcs = {
5003 .is_rlc_enabled = gfx_v9_0_is_rlc_enabled,
5004 .set_safe_mode = gfx_v9_0_set_safe_mode,
5005 .unset_safe_mode = gfx_v9_0_unset_safe_mode,
5006 .init = gfx_v9_0_rlc_init,
5007 .get_csb_size = gfx_v9_0_get_csb_size,
5008 .get_csb_buffer = gfx_v9_0_get_csb_buffer,
5009 .get_cp_table_num = gfx_v9_0_cp_jump_table_num,
5010 .resume = gfx_v9_0_rlc_resume,
5011 .stop = gfx_v9_0_rlc_stop,
5012 .reset = gfx_v9_0_rlc_reset,
5013 .start = gfx_v9_0_rlc_start,
5014 .update_spm_vmid = gfx_v9_0_update_spm_vmid,
5015 .rlcg_wreg = gfx_v9_0_rlcg_wreg,
5016 .is_rlcg_access_range = gfx_v9_0_is_rlcg_access_range,
5019 static int gfx_v9_0_set_powergating_state(void *handle,
5020 enum amd_powergating_state state)
5022 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5023 bool enable = (state == AMD_PG_STATE_GATE);
5025 switch (adev->asic_type) {
5029 amdgpu_gfx_off_ctrl(adev, false);
5030 cancel_delayed_work_sync(&adev->gfx.gfx_off_delay_work);
5032 if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) {
5033 gfx_v9_0_enable_sck_slow_down_on_power_up(adev, true);
5034 gfx_v9_0_enable_sck_slow_down_on_power_down(adev, true);
5036 gfx_v9_0_enable_sck_slow_down_on_power_up(adev, false);
5037 gfx_v9_0_enable_sck_slow_down_on_power_down(adev, false);
5040 if (adev->pg_flags & AMD_PG_SUPPORT_CP)
5041 gfx_v9_0_enable_cp_power_gating(adev, true);
5043 gfx_v9_0_enable_cp_power_gating(adev, false);
5045 /* update gfx cgpg state */
5046 gfx_v9_0_update_gfx_cg_power_gating(adev, enable);
5048 /* update mgcg state */
5049 gfx_v9_0_update_gfx_mg_power_gating(adev, enable);
5052 amdgpu_gfx_off_ctrl(adev, true);
5056 amdgpu_gfx_off_ctrl(adev, false);
5057 cancel_delayed_work_sync(&adev->gfx.gfx_off_delay_work);
5059 amdgpu_gfx_off_ctrl(adev, true);
5069 static int gfx_v9_0_set_clockgating_state(void *handle,
5070 enum amd_clockgating_state state)
5072 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5074 if (amdgpu_sriov_vf(adev))
5077 switch (adev->asic_type) {
5084 gfx_v9_0_update_gfx_clock_gating(adev,
5085 state == AMD_CG_STATE_GATE);
5093 static void gfx_v9_0_get_clockgating_state(void *handle, u32 *flags)
5095 struct amdgpu_device *adev = (struct amdgpu_device *)handle;
5098 if (amdgpu_sriov_vf(adev))
5101 /* AMD_CG_SUPPORT_GFX_MGCG */
5102 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGTT_MGCG_OVERRIDE));
5103 if (!(data & RLC_CGTT_MGCG_OVERRIDE__GFXIP_MGCG_OVERRIDE_MASK))
5104 *flags |= AMD_CG_SUPPORT_GFX_MGCG;
5106 /* AMD_CG_SUPPORT_GFX_CGCG */
5107 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL));
5108 if (data & RLC_CGCG_CGLS_CTRL__CGCG_EN_MASK)
5109 *flags |= AMD_CG_SUPPORT_GFX_CGCG;
5111 /* AMD_CG_SUPPORT_GFX_CGLS */
5112 if (data & RLC_CGCG_CGLS_CTRL__CGLS_EN_MASK)
5113 *flags |= AMD_CG_SUPPORT_GFX_CGLS;
5115 /* AMD_CG_SUPPORT_GFX_RLC_LS */
5116 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_MEM_SLP_CNTL));
5117 if (data & RLC_MEM_SLP_CNTL__RLC_MEM_LS_EN_MASK)
5118 *flags |= AMD_CG_SUPPORT_GFX_RLC_LS | AMD_CG_SUPPORT_GFX_MGLS;
5120 /* AMD_CG_SUPPORT_GFX_CP_LS */
5121 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmCP_MEM_SLP_CNTL));
5122 if (data & CP_MEM_SLP_CNTL__CP_MEM_LS_EN_MASK)
5123 *flags |= AMD_CG_SUPPORT_GFX_CP_LS | AMD_CG_SUPPORT_GFX_MGLS;
5125 if (adev->asic_type != CHIP_ARCTURUS) {
5126 /* AMD_CG_SUPPORT_GFX_3D_CGCG */
5127 data = RREG32_KIQ(SOC15_REG_OFFSET(GC, 0, mmRLC_CGCG_CGLS_CTRL_3D));
5128 if (data & RLC_CGCG_CGLS_CTRL_3D__CGCG_EN_MASK)
5129 *flags |= AMD_CG_SUPPORT_GFX_3D_CGCG;
5131 /* AMD_CG_SUPPORT_GFX_3D_CGLS */
5132 if (data & RLC_CGCG_CGLS_CTRL_3D__CGLS_EN_MASK)
5133 *flags |= AMD_CG_SUPPORT_GFX_3D_CGLS;
5137 static u64 gfx_v9_0_ring_get_rptr_gfx(struct amdgpu_ring *ring)
5139 return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 is 32bit rptr*/
5142 static u64 gfx_v9_0_ring_get_wptr_gfx(struct amdgpu_ring *ring)
5144 struct amdgpu_device *adev = ring->adev;
5147 /* XXX check if swapping is necessary on BE */
5148 if (ring->use_doorbell) {
5149 wptr = atomic64_read((atomic64_t *)&adev->wb.wb[ring->wptr_offs]);
5151 wptr = RREG32_SOC15(GC, 0, mmCP_RB0_WPTR);
5152 wptr += (u64)RREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI) << 32;
5158 static void gfx_v9_0_ring_set_wptr_gfx(struct amdgpu_ring *ring)
5160 struct amdgpu_device *adev = ring->adev;
5162 if (ring->use_doorbell) {
5163 /* XXX check if swapping is necessary on BE */
5164 atomic64_set((atomic64_t*)&adev->wb.wb[ring->wptr_offs], ring->wptr);
5165 WDOORBELL64(ring->doorbell_index, ring->wptr);
5167 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR, lower_32_bits(ring->wptr));
5168 WREG32_SOC15(GC, 0, mmCP_RB0_WPTR_HI, upper_32_bits(ring->wptr));
5172 static void gfx_v9_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
5174 struct amdgpu_device *adev = ring->adev;
5175 u32 ref_and_mask, reg_mem_engine;
5176 const struct nbio_hdp_flush_reg *nbio_hf_reg = adev->nbio.hdp_flush_reg;
5178 if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) {
5181 ref_and_mask = nbio_hf_reg->ref_and_mask_cp2 << ring->pipe;
5184 ref_and_mask = nbio_hf_reg->ref_and_mask_cp6 << ring->pipe;
5191 ref_and_mask = nbio_hf_reg->ref_and_mask_cp0;
5192 reg_mem_engine = 1; /* pfp */
5195 gfx_v9_0_wait_reg_mem(ring, reg_mem_engine, 0, 1,
5196 adev->nbio.funcs->get_hdp_flush_req_offset(adev),
5197 adev->nbio.funcs->get_hdp_flush_done_offset(adev),
5198 ref_and_mask, ref_and_mask, 0x20);
5201 static void gfx_v9_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
5202 struct amdgpu_job *job,
5203 struct amdgpu_ib *ib,
5206 unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5207 u32 header, control = 0;
5209 if (ib->flags & AMDGPU_IB_FLAG_CE)
5210 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
5212 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
5214 control |= ib->length_dw | (vmid << 24);
5216 if (amdgpu_sriov_vf(ring->adev) && (ib->flags & AMDGPU_IB_FLAG_PREEMPT)) {
5217 control |= INDIRECT_BUFFER_PRE_ENB(1);
5219 if (!(ib->flags & AMDGPU_IB_FLAG_CE) && vmid)
5220 gfx_v9_0_ring_emit_de_meta(ring);
5223 amdgpu_ring_write(ring, header);
5224 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5225 amdgpu_ring_write(ring,
5229 lower_32_bits(ib->gpu_addr));
5230 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5231 amdgpu_ring_write(ring, control);
5234 static void gfx_v9_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
5235 struct amdgpu_job *job,
5236 struct amdgpu_ib *ib,
5239 unsigned vmid = AMDGPU_JOB_GET_VMID(job);
5240 u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
5242 /* Currently, there is a high possibility to get wave ID mismatch
5243 * between ME and GDS, leading to a hw deadlock, because ME generates
5244 * different wave IDs than the GDS expects. This situation happens
5245 * randomly when at least 5 compute pipes use GDS ordered append.
5246 * The wave IDs generated by ME are also wrong after suspend/resume.
5247 * Those are probably bugs somewhere else in the kernel driver.
5249 * Writing GDS_COMPUTE_MAX_WAVE_ID resets wave ID counters in ME and
5250 * GDS to 0 for this ring (me/pipe).
5252 if (ib->flags & AMDGPU_IB_FLAG_RESET_GDS_MAX_WAVE_ID) {
5253 amdgpu_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
5254 amdgpu_ring_write(ring, mmGDS_COMPUTE_MAX_WAVE_ID);
5255 amdgpu_ring_write(ring, ring->adev->gds.gds_compute_max_wave_id);
5258 amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
5259 BUG_ON(ib->gpu_addr & 0x3); /* Dword align */
5260 amdgpu_ring_write(ring,
5264 lower_32_bits(ib->gpu_addr));
5265 amdgpu_ring_write(ring, upper_32_bits(ib->gpu_addr));
5266 amdgpu_ring_write(ring, control);
5269 static void gfx_v9_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
5270 u64 seq, unsigned flags)
5272 bool write64bit = flags & AMDGPU_FENCE_FLAG_64BIT;
5273 bool int_sel = flags & AMDGPU_FENCE_FLAG_INT;
5274 bool writeback = flags & AMDGPU_FENCE_FLAG_TC_WB_ONLY;
5276 /* RELEASE_MEM - flush caches, send int */
5277 amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));
5278 amdgpu_ring_write(ring, ((writeback ? (EOP_TC_WB_ACTION_EN |
5279 EOP_TC_NC_ACTION_EN) :
5280 (EOP_TCL1_ACTION_EN |
5282 EOP_TC_WB_ACTION_EN |
5283 EOP_TC_MD_ACTION_EN)) |
5284 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
5286 amdgpu_ring_write(ring, DATA_SEL(write64bit ? 2 : 1) | INT_SEL(int_sel ? 2 : 0));
5289 * the address should be Qword aligned if 64bit write, Dword
5290 * aligned if only send 32bit data low (discard data high)
5296 amdgpu_ring_write(ring, lower_32_bits(addr));
5297 amdgpu_ring_write(ring, upper_32_bits(addr));
5298 amdgpu_ring_write(ring, lower_32_bits(seq));
5299 amdgpu_ring_write(ring, upper_32_bits(seq));
5300 amdgpu_ring_write(ring, 0);
5303 static void gfx_v9_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
5305 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5306 uint32_t seq = ring->fence_drv.sync_seq;
5307 uint64_t addr = ring->fence_drv.gpu_addr;
5309 gfx_v9_0_wait_reg_mem(ring, usepfp, 1, 0,
5310 lower_32_bits(addr), upper_32_bits(addr),
5311 seq, 0xffffffff, 4);
5314 static void gfx_v9_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
5315 unsigned vmid, uint64_t pd_addr)
5317 amdgpu_gmc_emit_flush_gpu_tlb(ring, vmid, pd_addr);
5319 /* compute doesn't have PFP */
5320 if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) {
5321 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5322 amdgpu_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5323 amdgpu_ring_write(ring, 0x0);
5327 static u64 gfx_v9_0_ring_get_rptr_compute(struct amdgpu_ring *ring)
5329 return ring->adev->wb.wb[ring->rptr_offs]; /* gfx9 hardware is 32bit rptr */
5332 static u64 gfx_v9_0_ring_get_wptr_compute(struct amdgpu_ring *ring)
5336 /* XXX check if swapping is necessary on BE */
5337 if (ring->use_doorbell)
5338 wptr = atomic64_read((atomic64_t *)&ring->adev->wb.wb[ring->wptr_offs]);
5344 static void gfx_v9_0_ring_set_wptr_compute(struct amdgpu_ring *ring)
5346 struct amdgpu_device *adev = ring->adev;
5348 /* XXX check if swapping is necessary on BE */
5349 if (ring->use_doorbell) {
5350 atomic64_set((atomic64_t*)&adev->wb.wb[ring->wptr_offs], ring->wptr);
5351 WDOORBELL64(ring->doorbell_index, ring->wptr);
5353 BUG(); /* only DOORBELL method supported on gfx9 now */
5357 static void gfx_v9_0_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
5358 u64 seq, unsigned int flags)
5360 struct amdgpu_device *adev = ring->adev;
5362 /* we only allocate 32bit for each seq wb address */
5363 BUG_ON(flags & AMDGPU_FENCE_FLAG_64BIT);
5365 /* write fence seq to the "addr" */
5366 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5367 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5368 WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
5369 amdgpu_ring_write(ring, lower_32_bits(addr));
5370 amdgpu_ring_write(ring, upper_32_bits(addr));
5371 amdgpu_ring_write(ring, lower_32_bits(seq));
5373 if (flags & AMDGPU_FENCE_FLAG_INT) {
5374 /* set register to trigger INT */
5375 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5376 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5377 WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
5378 amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, 0, mmCPC_INT_STATUS));
5379 amdgpu_ring_write(ring, 0);
5380 amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
5384 static void gfx_v9_ring_emit_sb(struct amdgpu_ring *ring)
5386 amdgpu_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
5387 amdgpu_ring_write(ring, 0);
5390 static void gfx_v9_0_ring_emit_ce_meta(struct amdgpu_ring *ring)
5392 struct v9_ce_ib_state ce_payload = {0};
5396 cnt = (sizeof(ce_payload) >> 2) + 4 - 2;
5397 csa_addr = amdgpu_csa_vaddr(ring->adev);
5399 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
5400 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(2) |
5401 WRITE_DATA_DST_SEL(8) |
5403 WRITE_DATA_CACHE_POLICY(0));
5404 amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload)));
5405 amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, ce_payload)));
5406 amdgpu_ring_write_multiple(ring, (void *)&ce_payload, sizeof(ce_payload) >> 2);
5409 static void gfx_v9_0_ring_emit_de_meta(struct amdgpu_ring *ring)
5411 struct v9_de_ib_state de_payload = {0};
5412 uint64_t csa_addr, gds_addr;
5415 csa_addr = amdgpu_csa_vaddr(ring->adev);
5416 gds_addr = csa_addr + 4096;
5417 de_payload.gds_backup_addrlo = lower_32_bits(gds_addr);
5418 de_payload.gds_backup_addrhi = upper_32_bits(gds_addr);
5420 cnt = (sizeof(de_payload) >> 2) + 4 - 2;
5421 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, cnt));
5422 amdgpu_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5423 WRITE_DATA_DST_SEL(8) |
5425 WRITE_DATA_CACHE_POLICY(0));
5426 amdgpu_ring_write(ring, lower_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload)));
5427 amdgpu_ring_write(ring, upper_32_bits(csa_addr + offsetof(struct v9_gfx_meta_data, de_payload)));
5428 amdgpu_ring_write_multiple(ring, (void *)&de_payload, sizeof(de_payload) >> 2);
5431 static void gfx_v9_0_ring_emit_tmz(struct amdgpu_ring *ring, bool start)
5433 amdgpu_ring_write(ring, PACKET3(PACKET3_FRAME_CONTROL, 0));
5434 amdgpu_ring_write(ring, FRAME_CMD(start ? 0 : 1)); /* frame_end */
5437 static void gfx_v9_ring_emit_cntxcntl(struct amdgpu_ring *ring, uint32_t flags)
5441 if (amdgpu_sriov_vf(ring->adev))
5442 gfx_v9_0_ring_emit_ce_meta(ring);
5444 gfx_v9_0_ring_emit_tmz(ring, true);
5446 dw2 |= 0x80000000; /* set load_enable otherwise this package is just NOPs */
5447 if (flags & AMDGPU_HAVE_CTX_SWITCH) {
5448 /* set load_global_config & load_global_uconfig */
5450 /* set load_cs_sh_regs */
5452 /* set load_per_context_state & load_gfx_sh_regs for GFX */
5455 /* set load_ce_ram if preamble presented */
5456 if (AMDGPU_PREAMBLE_IB_PRESENT & flags)
5459 /* still load_ce_ram if this is the first time preamble presented
5460 * although there is no context switch happens.
5462 if (AMDGPU_PREAMBLE_IB_PRESENT_FIRST & flags)
5466 amdgpu_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5467 amdgpu_ring_write(ring, dw2);
5468 amdgpu_ring_write(ring, 0);
5471 static unsigned gfx_v9_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring)
5474 amdgpu_ring_write(ring, PACKET3(PACKET3_COND_EXEC, 3));
5475 amdgpu_ring_write(ring, lower_32_bits(ring->cond_exe_gpu_addr));
5476 amdgpu_ring_write(ring, upper_32_bits(ring->cond_exe_gpu_addr));
5477 amdgpu_ring_write(ring, 0); /* discard following DWs if *cond_exec_gpu_addr==0 */
5478 ret = ring->wptr & ring->buf_mask;
5479 amdgpu_ring_write(ring, 0x55aa55aa); /* patch dummy value later */
5483 static void gfx_v9_0_ring_emit_patch_cond_exec(struct amdgpu_ring *ring, unsigned offset)
5486 BUG_ON(offset > ring->buf_mask);
5487 BUG_ON(ring->ring[offset] != 0x55aa55aa);
5489 cur = (ring->wptr & ring->buf_mask) - 1;
5490 if (likely(cur > offset))
5491 ring->ring[offset] = cur - offset;
5493 ring->ring[offset] = (ring->ring_size>>2) - offset + cur;
5496 static void gfx_v9_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg)
5498 struct amdgpu_device *adev = ring->adev;
5499 struct amdgpu_kiq *kiq = &adev->gfx.kiq;
5501 amdgpu_ring_write(ring, PACKET3(PACKET3_COPY_DATA, 4));
5502 amdgpu_ring_write(ring, 0 | /* src: register*/
5503 (5 << 8) | /* dst: memory */
5504 (1 << 20)); /* write confirm */
5505 amdgpu_ring_write(ring, reg);
5506 amdgpu_ring_write(ring, 0);
5507 amdgpu_ring_write(ring, lower_32_bits(adev->wb.gpu_addr +
5508 kiq->reg_val_offs * 4));
5509 amdgpu_ring_write(ring, upper_32_bits(adev->wb.gpu_addr +
5510 kiq->reg_val_offs * 4));
5513 static void gfx_v9_0_ring_emit_wreg(struct amdgpu_ring *ring, uint32_t reg,
5518 switch (ring->funcs->type) {
5519 case AMDGPU_RING_TYPE_GFX:
5520 cmd = WRITE_DATA_ENGINE_SEL(1) | WR_CONFIRM;
5522 case AMDGPU_RING_TYPE_KIQ:
5523 cmd = (1 << 16); /* no inc addr */
5529 amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5530 amdgpu_ring_write(ring, cmd);
5531 amdgpu_ring_write(ring, reg);
5532 amdgpu_ring_write(ring, 0);
5533 amdgpu_ring_write(ring, val);
5536 static void gfx_v9_0_ring_emit_reg_wait(struct amdgpu_ring *ring, uint32_t reg,
5537 uint32_t val, uint32_t mask)
5539 gfx_v9_0_wait_reg_mem(ring, 0, 0, 0, reg, 0, val, mask, 0x20);
5542 static void gfx_v9_0_ring_emit_reg_write_reg_wait(struct amdgpu_ring *ring,
5543 uint32_t reg0, uint32_t reg1,
5544 uint32_t ref, uint32_t mask)
5546 int usepfp = (ring->funcs->type == AMDGPU_RING_TYPE_GFX);
5547 struct amdgpu_device *adev = ring->adev;
5548 bool fw_version_ok = (ring->funcs->type == AMDGPU_RING_TYPE_GFX) ?
5549 adev->gfx.me_fw_write_wait : adev->gfx.mec_fw_write_wait;
5552 gfx_v9_0_wait_reg_mem(ring, usepfp, 0, 1, reg0, reg1,
5555 amdgpu_ring_emit_reg_write_reg_wait_helper(ring, reg0, reg1,
5559 static void gfx_v9_0_ring_soft_recovery(struct amdgpu_ring *ring, unsigned vmid)
5561 struct amdgpu_device *adev = ring->adev;
5564 value = REG_SET_FIELD(value, SQ_CMD, CMD, 0x03);
5565 value = REG_SET_FIELD(value, SQ_CMD, MODE, 0x01);
5566 value = REG_SET_FIELD(value, SQ_CMD, CHECK_VMID, 1);
5567 value = REG_SET_FIELD(value, SQ_CMD, VM_ID, vmid);
5568 WREG32_SOC15(GC, 0, mmSQ_CMD, value);
5571 static void gfx_v9_0_set_gfx_eop_interrupt_state(struct amdgpu_device *adev,
5572 enum amdgpu_interrupt_state state)
5575 case AMDGPU_IRQ_STATE_DISABLE:
5576 case AMDGPU_IRQ_STATE_ENABLE:
5577 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5578 TIME_STAMP_INT_ENABLE,
5579 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5586 static void gfx_v9_0_set_compute_eop_interrupt_state(struct amdgpu_device *adev,
5588 enum amdgpu_interrupt_state state)
5590 u32 mec_int_cntl, mec_int_cntl_reg;
5593 * amdgpu controls only the first MEC. That's why this function only
5594 * handles the setting of interrupts for this specific MEC. All other
5595 * pipes' interrupts are set by amdkfd.
5601 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE0_INT_CNTL);
5604 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE1_INT_CNTL);
5607 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE2_INT_CNTL);
5610 mec_int_cntl_reg = SOC15_REG_OFFSET(GC, 0, mmCP_ME1_PIPE3_INT_CNTL);
5613 DRM_DEBUG("invalid pipe %d\n", pipe);
5617 DRM_DEBUG("invalid me %d\n", me);
5622 case AMDGPU_IRQ_STATE_DISABLE:
5623 mec_int_cntl = RREG32(mec_int_cntl_reg);
5624 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5625 TIME_STAMP_INT_ENABLE, 0);
5626 WREG32(mec_int_cntl_reg, mec_int_cntl);
5628 case AMDGPU_IRQ_STATE_ENABLE:
5629 mec_int_cntl = RREG32(mec_int_cntl_reg);
5630 mec_int_cntl = REG_SET_FIELD(mec_int_cntl, CP_ME1_PIPE0_INT_CNTL,
5631 TIME_STAMP_INT_ENABLE, 1);
5632 WREG32(mec_int_cntl_reg, mec_int_cntl);
5639 static int gfx_v9_0_set_priv_reg_fault_state(struct amdgpu_device *adev,
5640 struct amdgpu_irq_src *source,
5642 enum amdgpu_interrupt_state state)
5645 case AMDGPU_IRQ_STATE_DISABLE:
5646 case AMDGPU_IRQ_STATE_ENABLE:
5647 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5648 PRIV_REG_INT_ENABLE,
5649 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5658 static int gfx_v9_0_set_priv_inst_fault_state(struct amdgpu_device *adev,
5659 struct amdgpu_irq_src *source,
5661 enum amdgpu_interrupt_state state)
5664 case AMDGPU_IRQ_STATE_DISABLE:
5665 case AMDGPU_IRQ_STATE_ENABLE:
5666 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5667 PRIV_INSTR_INT_ENABLE,
5668 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
5676 #define ENABLE_ECC_ON_ME_PIPE(me, pipe) \
5677 WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\
5678 CP_ECC_ERROR_INT_ENABLE, 1)
5680 #define DISABLE_ECC_ON_ME_PIPE(me, pipe) \
5681 WREG32_FIELD15(GC, 0, CP_ME##me##_PIPE##pipe##_INT_CNTL,\
5682 CP_ECC_ERROR_INT_ENABLE, 0)
5684 static int gfx_v9_0_set_cp_ecc_error_state(struct amdgpu_device *adev,
5685 struct amdgpu_irq_src *source,
5687 enum amdgpu_interrupt_state state)
5690 case AMDGPU_IRQ_STATE_DISABLE:
5691 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5692 CP_ECC_ERROR_INT_ENABLE, 0);
5693 DISABLE_ECC_ON_ME_PIPE(1, 0);
5694 DISABLE_ECC_ON_ME_PIPE(1, 1);
5695 DISABLE_ECC_ON_ME_PIPE(1, 2);
5696 DISABLE_ECC_ON_ME_PIPE(1, 3);
5699 case AMDGPU_IRQ_STATE_ENABLE:
5700 WREG32_FIELD15(GC, 0, CP_INT_CNTL_RING0,
5701 CP_ECC_ERROR_INT_ENABLE, 1);
5702 ENABLE_ECC_ON_ME_PIPE(1, 0);
5703 ENABLE_ECC_ON_ME_PIPE(1, 1);
5704 ENABLE_ECC_ON_ME_PIPE(1, 2);
5705 ENABLE_ECC_ON_ME_PIPE(1, 3);
5715 static int gfx_v9_0_set_eop_interrupt_state(struct amdgpu_device *adev,
5716 struct amdgpu_irq_src *src,
5718 enum amdgpu_interrupt_state state)
5721 case AMDGPU_CP_IRQ_GFX_ME0_PIPE0_EOP:
5722 gfx_v9_0_set_gfx_eop_interrupt_state(adev, state);
5724 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP:
5725 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 0, state);
5727 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE1_EOP:
5728 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 1, state);
5730 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE2_EOP:
5731 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 2, state);
5733 case AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE3_EOP:
5734 gfx_v9_0_set_compute_eop_interrupt_state(adev, 1, 3, state);
5736 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE0_EOP:
5737 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 0, state);
5739 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE1_EOP:
5740 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 1, state);
5742 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE2_EOP:
5743 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 2, state);
5745 case AMDGPU_CP_IRQ_COMPUTE_MEC2_PIPE3_EOP:
5746 gfx_v9_0_set_compute_eop_interrupt_state(adev, 2, 3, state);
5754 static int gfx_v9_0_eop_irq(struct amdgpu_device *adev,
5755 struct amdgpu_irq_src *source,
5756 struct amdgpu_iv_entry *entry)
5759 u8 me_id, pipe_id, queue_id;
5760 struct amdgpu_ring *ring;
5762 DRM_DEBUG("IH: CP EOP\n");
5763 me_id = (entry->ring_id & 0x0c) >> 2;
5764 pipe_id = (entry->ring_id & 0x03) >> 0;
5765 queue_id = (entry->ring_id & 0x70) >> 4;
5769 amdgpu_fence_process(&adev->gfx.gfx_ring[0]);
5773 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5774 ring = &adev->gfx.compute_ring[i];
5775 /* Per-queue interrupt is supported for MEC starting from VI.
5776 * The interrupt can only be enabled/disabled per pipe instead of per queue.
5778 if ((ring->me == me_id) && (ring->pipe == pipe_id) && (ring->queue == queue_id))
5779 amdgpu_fence_process(ring);
5786 static void gfx_v9_0_fault(struct amdgpu_device *adev,
5787 struct amdgpu_iv_entry *entry)
5789 u8 me_id, pipe_id, queue_id;
5790 struct amdgpu_ring *ring;
5793 me_id = (entry->ring_id & 0x0c) >> 2;
5794 pipe_id = (entry->ring_id & 0x03) >> 0;
5795 queue_id = (entry->ring_id & 0x70) >> 4;
5799 drm_sched_fault(&adev->gfx.gfx_ring[0].sched);
5803 for (i = 0; i < adev->gfx.num_compute_rings; i++) {
5804 ring = &adev->gfx.compute_ring[i];
5805 if (ring->me == me_id && ring->pipe == pipe_id &&
5806 ring->queue == queue_id)
5807 drm_sched_fault(&ring->sched);
5813 static int gfx_v9_0_priv_reg_irq(struct amdgpu_device *adev,
5814 struct amdgpu_irq_src *source,
5815 struct amdgpu_iv_entry *entry)
5817 DRM_ERROR("Illegal register access in command stream\n");
5818 gfx_v9_0_fault(adev, entry);
5822 static int gfx_v9_0_priv_inst_irq(struct amdgpu_device *adev,
5823 struct amdgpu_irq_src *source,
5824 struct amdgpu_iv_entry *entry)
5826 DRM_ERROR("Illegal instruction in command stream\n");
5827 gfx_v9_0_fault(adev, entry);
5832 static const struct soc15_ras_field_entry gfx_v9_0_ras_fields[] = {
5833 { "CPC_SCRATCH", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_SCRATCH_CNT),
5834 SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, SEC_COUNT),
5835 SOC15_REG_FIELD(CPC_EDC_SCRATCH_CNT, DED_COUNT)
5837 { "CPC_UCODE", SOC15_REG_ENTRY(GC, 0, mmCPC_EDC_UCODE_CNT),
5838 SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, SEC_COUNT),
5839 SOC15_REG_FIELD(CPC_EDC_UCODE_CNT, DED_COUNT)
5841 { "CPF_ROQ_ME1", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT),
5842 SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME1),
5845 { "CPF_ROQ_ME2", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_ROQ_CNT),
5846 SOC15_REG_FIELD(CPF_EDC_ROQ_CNT, COUNT_ME2),
5849 { "CPF_TAG", SOC15_REG_ENTRY(GC, 0, mmCPF_EDC_TAG_CNT),
5850 SOC15_REG_FIELD(CPF_EDC_TAG_CNT, SEC_COUNT),
5851 SOC15_REG_FIELD(CPF_EDC_TAG_CNT, DED_COUNT)
5853 { "CPG_DMA_ROQ", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT),
5854 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, ROQ_COUNT),
5857 { "CPG_DMA_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_DMA_CNT),
5858 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_SEC_COUNT),
5859 SOC15_REG_FIELD(CPG_EDC_DMA_CNT, TAG_DED_COUNT)
5861 { "CPG_TAG", SOC15_REG_ENTRY(GC, 0, mmCPG_EDC_TAG_CNT),
5862 SOC15_REG_FIELD(CPG_EDC_TAG_CNT, SEC_COUNT),
5863 SOC15_REG_FIELD(CPG_EDC_TAG_CNT, DED_COUNT)
5865 { "DC_CSINVOC", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_CSINVOC_CNT),
5866 SOC15_REG_FIELD(DC_EDC_CSINVOC_CNT, COUNT_ME1),
5869 { "DC_RESTORE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_RESTORE_CNT),
5870 SOC15_REG_FIELD(DC_EDC_RESTORE_CNT, COUNT_ME1),
5873 { "DC_STATE", SOC15_REG_ENTRY(GC, 0, mmDC_EDC_STATE_CNT),
5874 SOC15_REG_FIELD(DC_EDC_STATE_CNT, COUNT_ME1),
5877 { "GDS_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT),
5878 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_SEC),
5879 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_MEM_DED)
5881 { "GDS_INPUT_QUEUE", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_CNT),
5882 SOC15_REG_FIELD(GDS_EDC_CNT, GDS_INPUT_QUEUE_SED),
5885 { "GDS_ME0_CS_PIPE_MEM", SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
5886 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_SEC),
5887 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, ME0_CS_PIPE_MEM_DED)
5889 { "GDS_OA_PHY_PHY_CMD_RAM_MEM",
5890 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
5891 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_SEC),
5892 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_CMD_RAM_MEM_DED)
5894 { "GDS_OA_PHY_PHY_DATA_RAM_MEM",
5895 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PHY_CNT),
5896 SOC15_REG_FIELD(GDS_EDC_OA_PHY_CNT, PHY_DATA_RAM_MEM_SED),
5899 { "GDS_OA_PIPE_ME1_PIPE0_PIPE_MEM",
5900 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
5901 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_SEC),
5902 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE0_PIPE_MEM_DED)
5904 { "GDS_OA_PIPE_ME1_PIPE1_PIPE_MEM",
5905 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
5906 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_SEC),
5907 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE1_PIPE_MEM_DED)
5909 { "GDS_OA_PIPE_ME1_PIPE2_PIPE_MEM",
5910 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
5911 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_SEC),
5912 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE2_PIPE_MEM_DED)
5914 { "GDS_OA_PIPE_ME1_PIPE3_PIPE_MEM",
5915 SOC15_REG_ENTRY(GC, 0, mmGDS_EDC_OA_PIPE_CNT),
5916 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_SEC),
5917 SOC15_REG_FIELD(GDS_EDC_OA_PIPE_CNT, ME1_PIPE3_PIPE_MEM_DED)
5919 { "SPI_SR_MEM", SOC15_REG_ENTRY(GC, 0, mmSPI_EDC_CNT),
5920 SOC15_REG_FIELD(SPI_EDC_CNT, SPI_SR_MEM_SED_COUNT),
5923 { "TA_FS_DFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
5924 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_SEC_COUNT),
5925 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_DFIFO_DED_COUNT)
5927 { "TA_FS_AFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
5928 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_AFIFO_SED_COUNT),
5931 { "TA_FL_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
5932 SOC15_REG_FIELD(TA_EDC_CNT, TA_FL_LFIFO_SED_COUNT),
5935 { "TA_FX_LFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
5936 SOC15_REG_FIELD(TA_EDC_CNT, TA_FX_LFIFO_SED_COUNT),
5939 { "TA_FS_CFIFO", SOC15_REG_ENTRY(GC, 0, mmTA_EDC_CNT),
5940 SOC15_REG_FIELD(TA_EDC_CNT, TA_FS_CFIFO_SED_COUNT),
5943 { "TCA_HOLE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT),
5944 SOC15_REG_FIELD(TCA_EDC_CNT, HOLE_FIFO_SED_COUNT),
5947 { "TCA_REQ_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCA_EDC_CNT),
5948 SOC15_REG_FIELD(TCA_EDC_CNT, REQ_FIFO_SED_COUNT),
5951 { "TCC_CACHE_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5952 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_SEC_COUNT),
5953 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DATA_DED_COUNT)
5955 { "TCC_CACHE_DIRTY", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5956 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_SEC_COUNT),
5957 SOC15_REG_FIELD(TCC_EDC_CNT, CACHE_DIRTY_DED_COUNT)
5959 { "TCC_HIGH_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5960 SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_SEC_COUNT),
5961 SOC15_REG_FIELD(TCC_EDC_CNT, HIGH_RATE_TAG_DED_COUNT)
5963 { "TCC_LOW_RATE_TAG", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5964 SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_SEC_COUNT),
5965 SOC15_REG_FIELD(TCC_EDC_CNT, LOW_RATE_TAG_DED_COUNT)
5967 { "TCC_SRC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5968 SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_SEC_COUNT),
5969 SOC15_REG_FIELD(TCC_EDC_CNT, SRC_FIFO_DED_COUNT)
5971 { "TCC_IN_USE_DEC", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5972 SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_DEC_SED_COUNT),
5975 { "TCC_IN_USE_TRANSFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5976 SOC15_REG_FIELD(TCC_EDC_CNT, IN_USE_TRANSFER_SED_COUNT),
5979 { "TCC_LATENCY_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5980 SOC15_REG_FIELD(TCC_EDC_CNT, LATENCY_FIFO_SED_COUNT),
5983 { "TCC_RETURN_DATA", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5984 SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_DATA_SED_COUNT),
5987 { "TCC_RETURN_CONTROL", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5988 SOC15_REG_FIELD(TCC_EDC_CNT, RETURN_CONTROL_SED_COUNT),
5991 { "TCC_UC_ATOMIC_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT),
5992 SOC15_REG_FIELD(TCC_EDC_CNT, UC_ATOMIC_FIFO_SED_COUNT),
5995 { "TCC_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
5996 SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_RETURN_SED_COUNT),
5999 { "TCC_WRITE_CACHE_READ", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6000 SOC15_REG_FIELD(TCC_EDC_CNT2, WRITE_CACHE_READ_SED_COUNT),
6003 { "TCC_SRC_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6004 SOC15_REG_FIELD(TCC_EDC_CNT2, SRC_FIFO_NEXT_RAM_SED_COUNT),
6007 { "TCC_LATENCY_FIFO_NEXT_RAM", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6008 SOC15_REG_FIELD(TCC_EDC_CNT2, LATENCY_FIFO_NEXT_RAM_SED_COUNT),
6011 { "TCC_CACHE_TAG_PROBE_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6012 SOC15_REG_FIELD(TCC_EDC_CNT2, CACHE_TAG_PROBE_FIFO_SED_COUNT),
6015 { "TCC_WRRET_TAG_WRITE_RETURN", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6016 SOC15_REG_FIELD(TCC_EDC_CNT2, WRRET_TAG_WRITE_RETURN_SED_COUNT),
6019 { "TCC_ATOMIC_RETURN_BUFFER", SOC15_REG_ENTRY(GC, 0, mmTCC_EDC_CNT2),
6020 SOC15_REG_FIELD(TCC_EDC_CNT2, ATOMIC_RETURN_BUFFER_SED_COUNT),
6023 { "TCI_WRITE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCI_EDC_CNT),
6024 SOC15_REG_FIELD(TCI_EDC_CNT, WRITE_RAM_SED_COUNT),
6027 { "TCP_CACHE_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6028 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_SEC_COUNT),
6029 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CACHE_RAM_DED_COUNT)
6031 { "TCP_LFIFO_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6032 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_SEC_COUNT),
6033 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, LFIFO_RAM_DED_COUNT)
6035 { "TCP_CMD_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6036 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, CMD_FIFO_SED_COUNT),
6039 { "TCP_VM_FIFO", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6040 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, VM_FIFO_SEC_COUNT),
6043 { "TCP_DB_RAM", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6044 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, DB_RAM_SED_COUNT),
6047 { "TCP_UTCL1_LFIFO0", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6048 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_SEC_COUNT),
6049 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO0_DED_COUNT)
6051 { "TCP_UTCL1_LFIFO1", SOC15_REG_ENTRY(GC, 0, mmTCP_EDC_CNT_NEW),
6052 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_SEC_COUNT),
6053 SOC15_REG_FIELD(TCP_EDC_CNT_NEW, UTCL1_LFIFO1_DED_COUNT)
6055 { "TD_SS_FIFO_LO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6056 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_SEC_COUNT),
6057 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_LO_DED_COUNT)
6059 { "TD_SS_FIFO_HI", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6060 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_SEC_COUNT),
6061 SOC15_REG_FIELD(TD_EDC_CNT, SS_FIFO_HI_DED_COUNT)
6063 { "TD_CS_FIFO", SOC15_REG_ENTRY(GC, 0, mmTD_EDC_CNT),
6064 SOC15_REG_FIELD(TD_EDC_CNT, CS_FIFO_SED_COUNT),
6067 { "SQ_LDS_D", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6068 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_SEC_COUNT),
6069 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_D_DED_COUNT)
6071 { "SQ_LDS_I", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6072 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_SEC_COUNT),
6073 SOC15_REG_FIELD(SQ_EDC_CNT, LDS_I_DED_COUNT)
6075 { "SQ_SGPR", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6076 SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_SEC_COUNT),
6077 SOC15_REG_FIELD(SQ_EDC_CNT, SGPR_DED_COUNT)
6079 { "SQ_VGPR0", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6080 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_SEC_COUNT),
6081 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR0_DED_COUNT)
6083 { "SQ_VGPR1", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6084 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_SEC_COUNT),
6085 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR1_DED_COUNT)
6087 { "SQ_VGPR2", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6088 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_SEC_COUNT),
6089 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR2_DED_COUNT)
6091 { "SQ_VGPR3", SOC15_REG_ENTRY(GC, 0, mmSQ_EDC_CNT),
6092 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_SEC_COUNT),
6093 SOC15_REG_FIELD(SQ_EDC_CNT, VGPR3_DED_COUNT)
6095 { "SQC_DATA_CU0_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6096 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_SEC_COUNT),
6097 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_WRITE_DATA_BUF_DED_COUNT)
6099 { "SQC_DATA_CU0_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6100 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_SEC_COUNT),
6101 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU0_UTCL1_LFIFO_DED_COUNT)
6103 { "SQC_DATA_CU1_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6104 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_SEC_COUNT),
6105 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_WRITE_DATA_BUF_DED_COUNT)
6107 { "SQC_DATA_CU1_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6108 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_SEC_COUNT),
6109 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU1_UTCL1_LFIFO_DED_COUNT)
6111 { "SQC_DATA_CU2_WRITE_DATA_BUF", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6112 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_SEC_COUNT),
6113 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_WRITE_DATA_BUF_DED_COUNT)
6115 { "SQC_DATA_CU2_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT),
6116 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_SEC_COUNT),
6117 SOC15_REG_FIELD(SQC_EDC_CNT, DATA_CU2_UTCL1_LFIFO_DED_COUNT)
6119 { "SQC_INST_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6120 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_SEC_COUNT),
6121 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_TAG_RAM_DED_COUNT)
6123 { "SQC_INST_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6124 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_SEC_COUNT),
6125 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_BANK_RAM_DED_COUNT)
6127 { "SQC_DATA_BANKA_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6128 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_SEC_COUNT),
6129 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_TAG_RAM_DED_COUNT)
6131 { "SQC_DATA_BANKA_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6132 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_SEC_COUNT),
6133 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_BANK_RAM_DED_COUNT)
6135 { "SQC_INST_BANKA_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6136 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_UTCL1_MISS_FIFO_SED_COUNT),
6139 { "SQC_INST_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6140 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_BANKA_MISS_FIFO_SED_COUNT),
6143 { "SQC_DATA_BANKA_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6144 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_HIT_FIFO_SED_COUNT),
6147 { "SQC_DATA_BANKA_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6148 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_MISS_FIFO_SED_COUNT),
6151 { "SQC_DATA_BANKA_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6152 SOC15_REG_FIELD(SQC_EDC_CNT2, DATA_BANKA_DIRTY_BIT_RAM_SED_COUNT),
6155 { "SQC_INST_UTCL1_LFIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT2),
6156 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_SEC_COUNT),
6157 SOC15_REG_FIELD(SQC_EDC_CNT2, INST_UTCL1_LFIFO_DED_COUNT)
6159 { "SQC_INST_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6160 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_SEC_COUNT),
6161 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_TAG_RAM_DED_COUNT)
6163 { "SQC_INST_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6164 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_SEC_COUNT),
6165 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_BANK_RAM_DED_COUNT)
6167 { "SQC_DATA_BANKB_TAG_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6168 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_SEC_COUNT),
6169 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_TAG_RAM_DED_COUNT)
6171 { "SQC_DATA_BANKB_BANK_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6172 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_SEC_COUNT),
6173 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_BANK_RAM_DED_COUNT)
6175 { "SQC_INST_BANKB_UTCL1_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6176 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_UTCL1_MISS_FIFO_SED_COUNT),
6179 { "SQC_INST_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6180 SOC15_REG_FIELD(SQC_EDC_CNT3, INST_BANKB_MISS_FIFO_SED_COUNT),
6183 { "SQC_DATA_BANKB_HIT_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6184 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_HIT_FIFO_SED_COUNT),
6187 { "SQC_DATA_BANKB_MISS_FIFO", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6188 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_MISS_FIFO_SED_COUNT),
6191 { "SQC_DATA_BANKB_DIRTY_BIT_RAM", SOC15_REG_ENTRY(GC, 0, mmSQC_EDC_CNT3),
6192 SOC15_REG_FIELD(SQC_EDC_CNT3, DATA_BANKB_DIRTY_BIT_RAM_SED_COUNT),
6195 { "EA_DRAMRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6196 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_SEC_COUNT),
6197 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_CMDMEM_DED_COUNT)
6199 { "EA_DRAMWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6200 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_SEC_COUNT),
6201 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_CMDMEM_DED_COUNT)
6203 { "EA_DRAMWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6204 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_SEC_COUNT),
6205 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_DATAMEM_DED_COUNT)
6207 { "EA_RRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6208 SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_SEC_COUNT),
6209 SOC15_REG_FIELD(GCEA_EDC_CNT, RRET_TAGMEM_DED_COUNT)
6211 { "EA_WRET_TAGMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6212 SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_SEC_COUNT),
6213 SOC15_REG_FIELD(GCEA_EDC_CNT, WRET_TAGMEM_DED_COUNT)
6215 { "EA_DRAMRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6216 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMRD_PAGEMEM_SED_COUNT),
6219 { "EA_DRAMWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6220 SOC15_REG_FIELD(GCEA_EDC_CNT, DRAMWR_PAGEMEM_SED_COUNT),
6223 { "EA_IORD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6224 SOC15_REG_FIELD(GCEA_EDC_CNT, IORD_CMDMEM_SED_COUNT),
6227 { "EA_IOWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6228 SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_CMDMEM_SED_COUNT),
6231 { "EA_IOWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT),
6232 SOC15_REG_FIELD(GCEA_EDC_CNT, IOWR_DATAMEM_SED_COUNT),
6235 { "GMIRD_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6236 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_SEC_COUNT),
6237 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_CMDMEM_DED_COUNT)
6239 { "GMIWR_CMDMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6240 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_SEC_COUNT),
6241 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_CMDMEM_DED_COUNT)
6243 { "GMIWR_DATAMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6244 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_SEC_COUNT),
6245 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_DATAMEM_DED_COUNT)
6247 { "GMIRD_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6248 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIRD_PAGEMEM_SED_COUNT),
6251 { "GMIWR_PAGEMEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6252 SOC15_REG_FIELD(GCEA_EDC_CNT2, GMIWR_PAGEMEM_SED_COUNT),
6255 { "MAM_D0MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6256 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D0MEM_SED_COUNT),
6259 { "MAM_D1MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6260 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D1MEM_SED_COUNT),
6263 { "MAM_D2MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6264 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D2MEM_SED_COUNT),
6267 { "MAM_D3MEM", SOC15_REG_ENTRY(GC, 0, mmGCEA_EDC_CNT2),
6268 SOC15_REG_FIELD(GCEA_EDC_CNT2, MAM_D3MEM_SED_COUNT),
6273 static int gfx_v9_0_ras_error_inject(struct amdgpu_device *adev,
6276 struct ras_inject_if *info = (struct ras_inject_if *)inject_if;
6278 struct ta_ras_trigger_error_input block_info = { 0 };
6280 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
6283 if (info->head.sub_block_index >= ARRAY_SIZE(ras_gfx_subblocks))
6286 if (!ras_gfx_subblocks[info->head.sub_block_index].name)
6289 if (!(ras_gfx_subblocks[info->head.sub_block_index].hw_supported_error_type &
6291 DRM_ERROR("GFX Subblock %s, hardware do not support type 0x%x\n",
6292 ras_gfx_subblocks[info->head.sub_block_index].name,
6297 if (!(ras_gfx_subblocks[info->head.sub_block_index].sw_supported_error_type &
6299 DRM_ERROR("GFX Subblock %s, driver do not support type 0x%x\n",
6300 ras_gfx_subblocks[info->head.sub_block_index].name,
6305 block_info.block_id = amdgpu_ras_block_to_ta(info->head.block);
6306 block_info.sub_block_index =
6307 ras_gfx_subblocks[info->head.sub_block_index].ta_subblock;
6308 block_info.inject_error_type = amdgpu_ras_error_to_ta(info->head.type);
6309 block_info.address = info->address;
6310 block_info.value = info->value;
6312 mutex_lock(&adev->grbm_idx_mutex);
6313 ret = psp_ras_trigger_error(&adev->psp, &block_info);
6314 mutex_unlock(&adev->grbm_idx_mutex);
6319 static const char *vml2_mems[] = {
6320 "UTC_VML2_BANK_CACHE_0_BIGK_MEM0",
6321 "UTC_VML2_BANK_CACHE_0_BIGK_MEM1",
6322 "UTC_VML2_BANK_CACHE_0_4K_MEM0",
6323 "UTC_VML2_BANK_CACHE_0_4K_MEM1",
6324 "UTC_VML2_BANK_CACHE_1_BIGK_MEM0",
6325 "UTC_VML2_BANK_CACHE_1_BIGK_MEM1",
6326 "UTC_VML2_BANK_CACHE_1_4K_MEM0",
6327 "UTC_VML2_BANK_CACHE_1_4K_MEM1",
6328 "UTC_VML2_BANK_CACHE_2_BIGK_MEM0",
6329 "UTC_VML2_BANK_CACHE_2_BIGK_MEM1",
6330 "UTC_VML2_BANK_CACHE_2_4K_MEM0",
6331 "UTC_VML2_BANK_CACHE_2_4K_MEM1",
6332 "UTC_VML2_BANK_CACHE_3_BIGK_MEM0",
6333 "UTC_VML2_BANK_CACHE_3_BIGK_MEM1",
6334 "UTC_VML2_BANK_CACHE_3_4K_MEM0",
6335 "UTC_VML2_BANK_CACHE_3_4K_MEM1",
6338 static const char *vml2_walker_mems[] = {
6339 "UTC_VML2_CACHE_PDE0_MEM0",
6340 "UTC_VML2_CACHE_PDE0_MEM1",
6341 "UTC_VML2_CACHE_PDE1_MEM0",
6342 "UTC_VML2_CACHE_PDE1_MEM1",
6343 "UTC_VML2_CACHE_PDE2_MEM0",
6344 "UTC_VML2_CACHE_PDE2_MEM1",
6345 "UTC_VML2_RDIF_LOG_FIFO",
6348 static const char *atc_l2_cache_2m_mems[] = {
6349 "UTC_ATCL2_CACHE_2M_BANK0_WAY0_MEM",
6350 "UTC_ATCL2_CACHE_2M_BANK0_WAY1_MEM",
6351 "UTC_ATCL2_CACHE_2M_BANK1_WAY0_MEM",
6352 "UTC_ATCL2_CACHE_2M_BANK1_WAY1_MEM",
6355 static const char *atc_l2_cache_4k_mems[] = {
6356 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM0",
6357 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM1",
6358 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM2",
6359 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM3",
6360 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM4",
6361 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM5",
6362 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM6",
6363 "UTC_ATCL2_CACHE_4K_BANK0_WAY0_MEM7",
6364 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM0",
6365 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM1",
6366 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM2",
6367 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM3",
6368 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM4",
6369 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM5",
6370 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM6",
6371 "UTC_ATCL2_CACHE_4K_BANK0_WAY1_MEM7",
6372 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM0",
6373 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM1",
6374 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM2",
6375 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM3",
6376 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM4",
6377 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM5",
6378 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM6",
6379 "UTC_ATCL2_CACHE_4K_BANK1_WAY0_MEM7",
6380 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM0",
6381 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM1",
6382 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM2",
6383 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM3",
6384 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM4",
6385 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM5",
6386 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM6",
6387 "UTC_ATCL2_CACHE_4K_BANK1_WAY1_MEM7",
6390 static int gfx_v9_0_query_utc_edc_status(struct amdgpu_device *adev,
6391 struct ras_err_data *err_data)
6394 uint32_t sec_count, ded_count;
6396 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6397 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0);
6398 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6399 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0);
6400 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6401 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0);
6402 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6403 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0);
6405 for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) {
6406 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i);
6407 data = RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT);
6409 sec_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, SEC_COUNT);
6411 DRM_INFO("Instance[%d]: SubBlock %s, SEC %d\n", i,
6412 vml2_mems[i], sec_count);
6413 err_data->ce_count += sec_count;
6416 ded_count = REG_GET_FIELD(data, VM_L2_MEM_ECC_CNT, DED_COUNT);
6418 DRM_INFO("Instance[%d]: SubBlock %s, DED %d\n", i,
6419 vml2_mems[i], ded_count);
6420 err_data->ue_count += ded_count;
6424 for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) {
6425 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i);
6426 data = RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT);
6428 sec_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT,
6431 DRM_INFO("Instance[%d]: SubBlock %s, SEC %d\n", i,
6432 vml2_walker_mems[i], sec_count);
6433 err_data->ce_count += sec_count;
6436 ded_count = REG_GET_FIELD(data, VM_L2_WALKER_MEM_ECC_CNT,
6439 DRM_INFO("Instance[%d]: SubBlock %s, DED %d\n", i,
6440 vml2_walker_mems[i], ded_count);
6441 err_data->ue_count += ded_count;
6445 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) {
6446 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i);
6447 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT);
6449 sec_count = (data & 0x00006000L) >> 0xd;
6451 DRM_INFO("Instance[%d]: SubBlock %s, SEC %d\n", i,
6452 atc_l2_cache_2m_mems[i], sec_count);
6453 err_data->ce_count += sec_count;
6457 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) {
6458 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i);
6459 data = RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT);
6461 sec_count = (data & 0x00006000L) >> 0xd;
6463 DRM_INFO("Instance[%d]: SubBlock %s, SEC %d\n", i,
6464 atc_l2_cache_4k_mems[i], sec_count);
6465 err_data->ce_count += sec_count;
6468 ded_count = (data & 0x00018000L) >> 0xf;
6470 DRM_INFO("Instance[%d]: SubBlock %s, DED %d\n", i,
6471 atc_l2_cache_4k_mems[i], ded_count);
6472 err_data->ue_count += ded_count;
6476 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6477 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6478 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6479 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6484 static int gfx_v9_0_ras_error_count(const struct soc15_reg_entry *reg,
6485 uint32_t se_id, uint32_t inst_id, uint32_t value,
6486 uint32_t *sec_count, uint32_t *ded_count)
6489 uint32_t sec_cnt, ded_cnt;
6491 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_ras_fields); i++) {
6492 if(gfx_v9_0_ras_fields[i].reg_offset != reg->reg_offset ||
6493 gfx_v9_0_ras_fields[i].seg != reg->seg ||
6494 gfx_v9_0_ras_fields[i].inst != reg->inst)
6498 gfx_v9_0_ras_fields[i].sec_count_mask) >>
6499 gfx_v9_0_ras_fields[i].sec_count_shift;
6501 DRM_INFO("GFX SubBlock %s, Instance[%d][%d], SEC %d\n",
6502 gfx_v9_0_ras_fields[i].name,
6505 *sec_count += sec_cnt;
6509 gfx_v9_0_ras_fields[i].ded_count_mask) >>
6510 gfx_v9_0_ras_fields[i].ded_count_shift;
6512 DRM_INFO("GFX SubBlock %s, Instance[%d][%d], DED %d\n",
6513 gfx_v9_0_ras_fields[i].name,
6516 *ded_count += ded_cnt;
6523 static void gfx_v9_0_reset_ras_error_count(struct amdgpu_device *adev)
6527 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
6530 /* read back registers to clear the counters */
6531 mutex_lock(&adev->grbm_idx_mutex);
6532 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) {
6533 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) {
6534 for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) {
6535 gfx_v9_0_select_se_sh(adev, j, 0x0, k);
6536 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i]));
6540 WREG32_SOC15(GC, 0, mmGRBM_GFX_INDEX, 0xe0000000);
6541 mutex_unlock(&adev->grbm_idx_mutex);
6543 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6544 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT, 0);
6545 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6546 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT, 0);
6547 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6548 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT, 0);
6549 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6550 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT, 0);
6552 for (i = 0; i < ARRAY_SIZE(vml2_mems); i++) {
6553 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, i);
6554 RREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_CNT);
6557 for (i = 0; i < ARRAY_SIZE(vml2_walker_mems); i++) {
6558 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, i);
6559 RREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_CNT);
6562 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_2m_mems); i++) {
6563 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, i);
6564 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_CNT);
6567 for (i = 0; i < ARRAY_SIZE(atc_l2_cache_4k_mems); i++) {
6568 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, i);
6569 RREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_CNT);
6572 WREG32_SOC15(GC, 0, mmVM_L2_MEM_ECC_INDEX, 255);
6573 WREG32_SOC15(GC, 0, mmVM_L2_WALKER_MEM_ECC_INDEX, 255);
6574 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_2M_EDC_INDEX, 255);
6575 WREG32_SOC15(GC, 0, mmATC_L2_CACHE_4K_EDC_INDEX, 255);
6578 static int gfx_v9_0_query_ras_error_count(struct amdgpu_device *adev,
6579 void *ras_error_status)
6581 struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status;
6582 uint32_t sec_count = 0, ded_count = 0;
6586 if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
6589 err_data->ue_count = 0;
6590 err_data->ce_count = 0;
6592 mutex_lock(&adev->grbm_idx_mutex);
6594 for (i = 0; i < ARRAY_SIZE(gfx_v9_0_edc_counter_regs); i++) {
6595 for (j = 0; j < gfx_v9_0_edc_counter_regs[i].se_num; j++) {
6596 for (k = 0; k < gfx_v9_0_edc_counter_regs[i].instance; k++) {
6597 gfx_v9_0_select_se_sh(adev, j, 0, k);
6599 RREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_0_edc_counter_regs[i]));
6601 gfx_v9_0_ras_error_count(&gfx_v9_0_edc_counter_regs[i],
6603 &sec_count, &ded_count);
6608 err_data->ce_count += sec_count;
6609 err_data->ue_count += ded_count;
6611 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
6612 mutex_unlock(&adev->grbm_idx_mutex);
6614 gfx_v9_0_query_utc_edc_status(adev, err_data);
6619 static const struct amd_ip_funcs gfx_v9_0_ip_funcs = {
6621 .early_init = gfx_v9_0_early_init,
6622 .late_init = gfx_v9_0_late_init,
6623 .sw_init = gfx_v9_0_sw_init,
6624 .sw_fini = gfx_v9_0_sw_fini,
6625 .hw_init = gfx_v9_0_hw_init,
6626 .hw_fini = gfx_v9_0_hw_fini,
6627 .suspend = gfx_v9_0_suspend,
6628 .resume = gfx_v9_0_resume,
6629 .is_idle = gfx_v9_0_is_idle,
6630 .wait_for_idle = gfx_v9_0_wait_for_idle,
6631 .soft_reset = gfx_v9_0_soft_reset,
6632 .set_clockgating_state = gfx_v9_0_set_clockgating_state,
6633 .set_powergating_state = gfx_v9_0_set_powergating_state,
6634 .get_clockgating_state = gfx_v9_0_get_clockgating_state,
6637 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_gfx = {
6638 .type = AMDGPU_RING_TYPE_GFX,
6640 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6641 .support_64bit_ptrs = true,
6642 .vmhub = AMDGPU_GFXHUB_0,
6643 .get_rptr = gfx_v9_0_ring_get_rptr_gfx,
6644 .get_wptr = gfx_v9_0_ring_get_wptr_gfx,
6645 .set_wptr = gfx_v9_0_ring_set_wptr_gfx,
6646 .emit_frame_size = /* totally 242 maximum if 16 IBs */
6648 7 + /* PIPELINE_SYNC */
6649 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6650 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6652 8 + /* FENCE for VM_FLUSH */
6653 20 + /* GDS switch */
6654 4 + /* double SWITCH_BUFFER,
6655 the first COND_EXEC jump to the place just
6656 prior to this double SWITCH_BUFFER */
6664 8 + 8 + /* FENCE x2 */
6665 2, /* SWITCH_BUFFER */
6666 .emit_ib_size = 4, /* gfx_v9_0_ring_emit_ib_gfx */
6667 .emit_ib = gfx_v9_0_ring_emit_ib_gfx,
6668 .emit_fence = gfx_v9_0_ring_emit_fence,
6669 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
6670 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
6671 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
6672 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
6673 .test_ring = gfx_v9_0_ring_test_ring,
6674 .test_ib = gfx_v9_0_ring_test_ib,
6675 .insert_nop = amdgpu_ring_insert_nop,
6676 .pad_ib = amdgpu_ring_generic_pad_ib,
6677 .emit_switch_buffer = gfx_v9_ring_emit_sb,
6678 .emit_cntxcntl = gfx_v9_ring_emit_cntxcntl,
6679 .init_cond_exec = gfx_v9_0_ring_emit_init_cond_exec,
6680 .patch_cond_exec = gfx_v9_0_ring_emit_patch_cond_exec,
6681 .emit_tmz = gfx_v9_0_ring_emit_tmz,
6682 .emit_wreg = gfx_v9_0_ring_emit_wreg,
6683 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
6684 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
6685 .soft_recovery = gfx_v9_0_ring_soft_recovery,
6688 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_compute = {
6689 .type = AMDGPU_RING_TYPE_COMPUTE,
6691 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6692 .support_64bit_ptrs = true,
6693 .vmhub = AMDGPU_GFXHUB_0,
6694 .get_rptr = gfx_v9_0_ring_get_rptr_compute,
6695 .get_wptr = gfx_v9_0_ring_get_wptr_compute,
6696 .set_wptr = gfx_v9_0_ring_set_wptr_compute,
6698 20 + /* gfx_v9_0_ring_emit_gds_switch */
6699 7 + /* gfx_v9_0_ring_emit_hdp_flush */
6700 5 + /* hdp invalidate */
6701 7 + /* gfx_v9_0_ring_emit_pipeline_sync */
6702 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6703 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6704 2 + /* gfx_v9_0_ring_emit_vm_flush */
6705 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence x3 for user fence, vm fence */
6706 .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */
6707 .emit_ib = gfx_v9_0_ring_emit_ib_compute,
6708 .emit_fence = gfx_v9_0_ring_emit_fence,
6709 .emit_pipeline_sync = gfx_v9_0_ring_emit_pipeline_sync,
6710 .emit_vm_flush = gfx_v9_0_ring_emit_vm_flush,
6711 .emit_gds_switch = gfx_v9_0_ring_emit_gds_switch,
6712 .emit_hdp_flush = gfx_v9_0_ring_emit_hdp_flush,
6713 .test_ring = gfx_v9_0_ring_test_ring,
6714 .test_ib = gfx_v9_0_ring_test_ib,
6715 .insert_nop = amdgpu_ring_insert_nop,
6716 .pad_ib = amdgpu_ring_generic_pad_ib,
6717 .emit_wreg = gfx_v9_0_ring_emit_wreg,
6718 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
6719 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
6722 static const struct amdgpu_ring_funcs gfx_v9_0_ring_funcs_kiq = {
6723 .type = AMDGPU_RING_TYPE_KIQ,
6725 .nop = PACKET3(PACKET3_NOP, 0x3FFF),
6726 .support_64bit_ptrs = true,
6727 .vmhub = AMDGPU_GFXHUB_0,
6728 .get_rptr = gfx_v9_0_ring_get_rptr_compute,
6729 .get_wptr = gfx_v9_0_ring_get_wptr_compute,
6730 .set_wptr = gfx_v9_0_ring_set_wptr_compute,
6732 20 + /* gfx_v9_0_ring_emit_gds_switch */
6733 7 + /* gfx_v9_0_ring_emit_hdp_flush */
6734 5 + /* hdp invalidate */
6735 7 + /* gfx_v9_0_ring_emit_pipeline_sync */
6736 SOC15_FLUSH_GPU_TLB_NUM_WREG * 5 +
6737 SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 7 +
6738 2 + /* gfx_v9_0_ring_emit_vm_flush */
6739 8 + 8 + 8, /* gfx_v9_0_ring_emit_fence_kiq x3 for user fence, vm fence */
6740 .emit_ib_size = 7, /* gfx_v9_0_ring_emit_ib_compute */
6741 .emit_fence = gfx_v9_0_ring_emit_fence_kiq,
6742 .test_ring = gfx_v9_0_ring_test_ring,
6743 .insert_nop = amdgpu_ring_insert_nop,
6744 .pad_ib = amdgpu_ring_generic_pad_ib,
6745 .emit_rreg = gfx_v9_0_ring_emit_rreg,
6746 .emit_wreg = gfx_v9_0_ring_emit_wreg,
6747 .emit_reg_wait = gfx_v9_0_ring_emit_reg_wait,
6748 .emit_reg_write_reg_wait = gfx_v9_0_ring_emit_reg_write_reg_wait,
6751 static void gfx_v9_0_set_ring_funcs(struct amdgpu_device *adev)
6755 adev->gfx.kiq.ring.funcs = &gfx_v9_0_ring_funcs_kiq;
6757 for (i = 0; i < adev->gfx.num_gfx_rings; i++)
6758 adev->gfx.gfx_ring[i].funcs = &gfx_v9_0_ring_funcs_gfx;
6760 for (i = 0; i < adev->gfx.num_compute_rings; i++)
6761 adev->gfx.compute_ring[i].funcs = &gfx_v9_0_ring_funcs_compute;
6764 static const struct amdgpu_irq_src_funcs gfx_v9_0_eop_irq_funcs = {
6765 .set = gfx_v9_0_set_eop_interrupt_state,
6766 .process = gfx_v9_0_eop_irq,
6769 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_reg_irq_funcs = {
6770 .set = gfx_v9_0_set_priv_reg_fault_state,
6771 .process = gfx_v9_0_priv_reg_irq,
6774 static const struct amdgpu_irq_src_funcs gfx_v9_0_priv_inst_irq_funcs = {
6775 .set = gfx_v9_0_set_priv_inst_fault_state,
6776 .process = gfx_v9_0_priv_inst_irq,
6779 static const struct amdgpu_irq_src_funcs gfx_v9_0_cp_ecc_error_irq_funcs = {
6780 .set = gfx_v9_0_set_cp_ecc_error_state,
6781 .process = amdgpu_gfx_cp_ecc_error_irq,
6785 static void gfx_v9_0_set_irq_funcs(struct amdgpu_device *adev)
6787 adev->gfx.eop_irq.num_types = AMDGPU_CP_IRQ_LAST;
6788 adev->gfx.eop_irq.funcs = &gfx_v9_0_eop_irq_funcs;
6790 adev->gfx.priv_reg_irq.num_types = 1;
6791 adev->gfx.priv_reg_irq.funcs = &gfx_v9_0_priv_reg_irq_funcs;
6793 adev->gfx.priv_inst_irq.num_types = 1;
6794 adev->gfx.priv_inst_irq.funcs = &gfx_v9_0_priv_inst_irq_funcs;
6796 adev->gfx.cp_ecc_error_irq.num_types = 2; /*C5 ECC error and C9 FUE error*/
6797 adev->gfx.cp_ecc_error_irq.funcs = &gfx_v9_0_cp_ecc_error_irq_funcs;
6800 static void gfx_v9_0_set_rlc_funcs(struct amdgpu_device *adev)
6802 switch (adev->asic_type) {
6809 adev->gfx.rlc.funcs = &gfx_v9_0_rlc_funcs;
6816 static void gfx_v9_0_set_gds_init(struct amdgpu_device *adev)
6818 /* init asci gds info */
6819 switch (adev->asic_type) {
6823 adev->gds.gds_size = 0x10000;
6827 adev->gds.gds_size = 0x1000;
6830 adev->gds.gds_size = 0x10000;
6834 switch (adev->asic_type) {
6837 adev->gds.gds_compute_max_wave_id = 0x7ff;
6840 adev->gds.gds_compute_max_wave_id = 0x27f;
6843 if (adev->rev_id >= 0x8)
6844 adev->gds.gds_compute_max_wave_id = 0x77; /* raven2 */
6846 adev->gds.gds_compute_max_wave_id = 0x15f; /* raven1 */
6849 adev->gds.gds_compute_max_wave_id = 0xfff;
6852 /* this really depends on the chip */
6853 adev->gds.gds_compute_max_wave_id = 0x7ff;
6857 adev->gds.gws_size = 64;
6858 adev->gds.oa_size = 16;
6861 static void gfx_v9_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev,
6869 data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
6870 data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
6872 WREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG, data);
6875 static u32 gfx_v9_0_get_cu_active_bitmap(struct amdgpu_device *adev)
6879 data = RREG32_SOC15(GC, 0, mmCC_GC_SHADER_ARRAY_CONFIG);
6880 data |= RREG32_SOC15(GC, 0, mmGC_USER_SHADER_ARRAY_CONFIG);
6882 data &= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
6883 data >>= CC_GC_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
6885 mask = amdgpu_gfx_create_bitmask(adev->gfx.config.max_cu_per_sh);
6887 return (~data) & mask;
6890 static int gfx_v9_0_get_cu_info(struct amdgpu_device *adev,
6891 struct amdgpu_cu_info *cu_info)
6893 int i, j, k, counter, active_cu_number = 0;
6894 u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0;
6895 unsigned disable_masks[4 * 4];
6897 if (!adev || !cu_info)
6901 * 16 comes from bitmap array size 4*4, and it can cover all gfx9 ASICs
6903 if (adev->gfx.config.max_shader_engines *
6904 adev->gfx.config.max_sh_per_se > 16)
6907 amdgpu_gfx_parse_disable_cu(disable_masks,
6908 adev->gfx.config.max_shader_engines,
6909 adev->gfx.config.max_sh_per_se);
6911 mutex_lock(&adev->grbm_idx_mutex);
6912 for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
6913 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
6917 gfx_v9_0_select_se_sh(adev, i, j, 0xffffffff);
6918 gfx_v9_0_set_user_cu_inactive_bitmap(
6919 adev, disable_masks[i * adev->gfx.config.max_sh_per_se + j]);
6920 bitmap = gfx_v9_0_get_cu_active_bitmap(adev);
6923 * The bitmap(and ao_cu_bitmap) in cu_info structure is
6924 * 4x4 size array, and it's usually suitable for Vega
6925 * ASICs which has 4*2 SE/SH layout.
6926 * But for Arcturus, SE/SH layout is changed to 8*1.
6927 * To mostly reduce the impact, we make it compatible
6928 * with current bitmap array as below:
6929 * SE4,SH0 --> bitmap[0][1]
6930 * SE5,SH0 --> bitmap[1][1]
6931 * SE6,SH0 --> bitmap[2][1]
6932 * SE7,SH0 --> bitmap[3][1]
6934 cu_info->bitmap[i % 4][j + i / 4] = bitmap;
6936 for (k = 0; k < adev->gfx.config.max_cu_per_sh; k ++) {
6937 if (bitmap & mask) {
6938 if (counter < adev->gfx.config.max_cu_per_sh)
6944 active_cu_number += counter;
6946 ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8));
6947 cu_info->ao_cu_bitmap[i % 4][j + i / 4] = ao_bitmap;
6950 gfx_v9_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
6951 mutex_unlock(&adev->grbm_idx_mutex);
6953 cu_info->number = active_cu_number;
6954 cu_info->ao_cu_mask = ao_cu_mask;
6955 cu_info->simd_per_cu = NUM_SIMD_PER_CU;
6960 const struct amdgpu_ip_block_version gfx_v9_0_ip_block =
6962 .type = AMD_IP_BLOCK_TYPE_GFX,
6966 .funcs = &gfx_v9_0_ip_funcs,