2 * Copyright 2012 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
22 * Authors: Alex Deucher
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
29 #include "radeon_asic.h"
32 #include "cik_blit_shaders.h"
33 #include "radeon_ucode.h"
34 #include "clearstate_ci.h"
36 MODULE_FIRMWARE("radeon/BONAIRE_pfp.bin");
37 MODULE_FIRMWARE("radeon/BONAIRE_me.bin");
38 MODULE_FIRMWARE("radeon/BONAIRE_ce.bin");
39 MODULE_FIRMWARE("radeon/BONAIRE_mec.bin");
40 MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
41 MODULE_FIRMWARE("radeon/BONAIRE_mc2.bin");
42 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
43 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
44 MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
45 MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
46 MODULE_FIRMWARE("radeon/HAWAII_me.bin");
47 MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
48 MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
49 MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
50 MODULE_FIRMWARE("radeon/HAWAII_mc2.bin");
51 MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
52 MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
53 MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
54 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
55 MODULE_FIRMWARE("radeon/KAVERI_me.bin");
56 MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
57 MODULE_FIRMWARE("radeon/KAVERI_mec.bin");
58 MODULE_FIRMWARE("radeon/KAVERI_rlc.bin");
59 MODULE_FIRMWARE("radeon/KAVERI_sdma.bin");
60 MODULE_FIRMWARE("radeon/KABINI_pfp.bin");
61 MODULE_FIRMWARE("radeon/KABINI_me.bin");
62 MODULE_FIRMWARE("radeon/KABINI_ce.bin");
63 MODULE_FIRMWARE("radeon/KABINI_mec.bin");
64 MODULE_FIRMWARE("radeon/KABINI_rlc.bin");
65 MODULE_FIRMWARE("radeon/KABINI_sdma.bin");
67 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
68 extern void r600_ih_ring_fini(struct radeon_device *rdev);
69 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
70 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
71 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
72 extern void sumo_rlc_fini(struct radeon_device *rdev);
73 extern int sumo_rlc_init(struct radeon_device *rdev);
74 extern void si_vram_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
75 extern void si_rlc_reset(struct radeon_device *rdev);
76 extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
77 extern int cik_sdma_resume(struct radeon_device *rdev);
78 extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
79 extern void cik_sdma_fini(struct radeon_device *rdev);
80 static void cik_rlc_stop(struct radeon_device *rdev);
81 static void cik_pcie_gen3_enable(struct radeon_device *rdev);
82 static void cik_program_aspm(struct radeon_device *rdev);
83 static void cik_init_pg(struct radeon_device *rdev);
84 static void cik_init_cg(struct radeon_device *rdev);
85 static void cik_fini_pg(struct radeon_device *rdev);
86 static void cik_fini_cg(struct radeon_device *rdev);
87 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
90 /* get temperature in millidegrees */
91 int ci_get_temp(struct radeon_device *rdev)
96 temp = (RREG32_SMC(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
102 actual_temp = temp & 0x1ff;
104 actual_temp = actual_temp * 1000;
109 /* get temperature in millidegrees */
110 int kv_get_temp(struct radeon_device *rdev)
115 temp = RREG32_SMC(0xC0300E0C);
118 actual_temp = (temp / 8) - 49;
122 actual_temp = actual_temp * 1000;
128 * Indirect registers accessor
130 u32 cik_pciep_rreg(struct radeon_device *rdev, u32 reg)
135 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
136 WREG32(PCIE_INDEX, reg);
137 (void)RREG32(PCIE_INDEX);
138 r = RREG32(PCIE_DATA);
139 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
143 void cik_pciep_wreg(struct radeon_device *rdev, u32 reg, u32 v)
147 spin_lock_irqsave(&rdev->pciep_idx_lock, flags);
148 WREG32(PCIE_INDEX, reg);
149 (void)RREG32(PCIE_INDEX);
150 WREG32(PCIE_DATA, v);
151 (void)RREG32(PCIE_DATA);
152 spin_unlock_irqrestore(&rdev->pciep_idx_lock, flags);
155 static const u32 spectre_rlc_save_restore_register_list[] =
157 (0x0e00 << 16) | (0xc12c >> 2),
159 (0x0e00 << 16) | (0xc140 >> 2),
161 (0x0e00 << 16) | (0xc150 >> 2),
163 (0x0e00 << 16) | (0xc15c >> 2),
165 (0x0e00 << 16) | (0xc168 >> 2),
167 (0x0e00 << 16) | (0xc170 >> 2),
169 (0x0e00 << 16) | (0xc178 >> 2),
171 (0x0e00 << 16) | (0xc204 >> 2),
173 (0x0e00 << 16) | (0xc2b4 >> 2),
175 (0x0e00 << 16) | (0xc2b8 >> 2),
177 (0x0e00 << 16) | (0xc2bc >> 2),
179 (0x0e00 << 16) | (0xc2c0 >> 2),
181 (0x0e00 << 16) | (0x8228 >> 2),
183 (0x0e00 << 16) | (0x829c >> 2),
185 (0x0e00 << 16) | (0x869c >> 2),
187 (0x0600 << 16) | (0x98f4 >> 2),
189 (0x0e00 << 16) | (0x98f8 >> 2),
191 (0x0e00 << 16) | (0x9900 >> 2),
193 (0x0e00 << 16) | (0xc260 >> 2),
195 (0x0e00 << 16) | (0x90e8 >> 2),
197 (0x0e00 << 16) | (0x3c000 >> 2),
199 (0x0e00 << 16) | (0x3c00c >> 2),
201 (0x0e00 << 16) | (0x8c1c >> 2),
203 (0x0e00 << 16) | (0x9700 >> 2),
205 (0x0e00 << 16) | (0xcd20 >> 2),
207 (0x4e00 << 16) | (0xcd20 >> 2),
209 (0x5e00 << 16) | (0xcd20 >> 2),
211 (0x6e00 << 16) | (0xcd20 >> 2),
213 (0x7e00 << 16) | (0xcd20 >> 2),
215 (0x8e00 << 16) | (0xcd20 >> 2),
217 (0x9e00 << 16) | (0xcd20 >> 2),
219 (0xae00 << 16) | (0xcd20 >> 2),
221 (0xbe00 << 16) | (0xcd20 >> 2),
223 (0x0e00 << 16) | (0x89bc >> 2),
225 (0x0e00 << 16) | (0x8900 >> 2),
228 (0x0e00 << 16) | (0xc130 >> 2),
230 (0x0e00 << 16) | (0xc134 >> 2),
232 (0x0e00 << 16) | (0xc1fc >> 2),
234 (0x0e00 << 16) | (0xc208 >> 2),
236 (0x0e00 << 16) | (0xc264 >> 2),
238 (0x0e00 << 16) | (0xc268 >> 2),
240 (0x0e00 << 16) | (0xc26c >> 2),
242 (0x0e00 << 16) | (0xc270 >> 2),
244 (0x0e00 << 16) | (0xc274 >> 2),
246 (0x0e00 << 16) | (0xc278 >> 2),
248 (0x0e00 << 16) | (0xc27c >> 2),
250 (0x0e00 << 16) | (0xc280 >> 2),
252 (0x0e00 << 16) | (0xc284 >> 2),
254 (0x0e00 << 16) | (0xc288 >> 2),
256 (0x0e00 << 16) | (0xc28c >> 2),
258 (0x0e00 << 16) | (0xc290 >> 2),
260 (0x0e00 << 16) | (0xc294 >> 2),
262 (0x0e00 << 16) | (0xc298 >> 2),
264 (0x0e00 << 16) | (0xc29c >> 2),
266 (0x0e00 << 16) | (0xc2a0 >> 2),
268 (0x0e00 << 16) | (0xc2a4 >> 2),
270 (0x0e00 << 16) | (0xc2a8 >> 2),
272 (0x0e00 << 16) | (0xc2ac >> 2),
274 (0x0e00 << 16) | (0xc2b0 >> 2),
276 (0x0e00 << 16) | (0x301d0 >> 2),
278 (0x0e00 << 16) | (0x30238 >> 2),
280 (0x0e00 << 16) | (0x30250 >> 2),
282 (0x0e00 << 16) | (0x30254 >> 2),
284 (0x0e00 << 16) | (0x30258 >> 2),
286 (0x0e00 << 16) | (0x3025c >> 2),
288 (0x4e00 << 16) | (0xc900 >> 2),
290 (0x5e00 << 16) | (0xc900 >> 2),
292 (0x6e00 << 16) | (0xc900 >> 2),
294 (0x7e00 << 16) | (0xc900 >> 2),
296 (0x8e00 << 16) | (0xc900 >> 2),
298 (0x9e00 << 16) | (0xc900 >> 2),
300 (0xae00 << 16) | (0xc900 >> 2),
302 (0xbe00 << 16) | (0xc900 >> 2),
304 (0x4e00 << 16) | (0xc904 >> 2),
306 (0x5e00 << 16) | (0xc904 >> 2),
308 (0x6e00 << 16) | (0xc904 >> 2),
310 (0x7e00 << 16) | (0xc904 >> 2),
312 (0x8e00 << 16) | (0xc904 >> 2),
314 (0x9e00 << 16) | (0xc904 >> 2),
316 (0xae00 << 16) | (0xc904 >> 2),
318 (0xbe00 << 16) | (0xc904 >> 2),
320 (0x4e00 << 16) | (0xc908 >> 2),
322 (0x5e00 << 16) | (0xc908 >> 2),
324 (0x6e00 << 16) | (0xc908 >> 2),
326 (0x7e00 << 16) | (0xc908 >> 2),
328 (0x8e00 << 16) | (0xc908 >> 2),
330 (0x9e00 << 16) | (0xc908 >> 2),
332 (0xae00 << 16) | (0xc908 >> 2),
334 (0xbe00 << 16) | (0xc908 >> 2),
336 (0x4e00 << 16) | (0xc90c >> 2),
338 (0x5e00 << 16) | (0xc90c >> 2),
340 (0x6e00 << 16) | (0xc90c >> 2),
342 (0x7e00 << 16) | (0xc90c >> 2),
344 (0x8e00 << 16) | (0xc90c >> 2),
346 (0x9e00 << 16) | (0xc90c >> 2),
348 (0xae00 << 16) | (0xc90c >> 2),
350 (0xbe00 << 16) | (0xc90c >> 2),
352 (0x4e00 << 16) | (0xc910 >> 2),
354 (0x5e00 << 16) | (0xc910 >> 2),
356 (0x6e00 << 16) | (0xc910 >> 2),
358 (0x7e00 << 16) | (0xc910 >> 2),
360 (0x8e00 << 16) | (0xc910 >> 2),
362 (0x9e00 << 16) | (0xc910 >> 2),
364 (0xae00 << 16) | (0xc910 >> 2),
366 (0xbe00 << 16) | (0xc910 >> 2),
368 (0x0e00 << 16) | (0xc99c >> 2),
370 (0x0e00 << 16) | (0x9834 >> 2),
372 (0x0000 << 16) | (0x30f00 >> 2),
374 (0x0001 << 16) | (0x30f00 >> 2),
376 (0x0000 << 16) | (0x30f04 >> 2),
378 (0x0001 << 16) | (0x30f04 >> 2),
380 (0x0000 << 16) | (0x30f08 >> 2),
382 (0x0001 << 16) | (0x30f08 >> 2),
384 (0x0000 << 16) | (0x30f0c >> 2),
386 (0x0001 << 16) | (0x30f0c >> 2),
388 (0x0600 << 16) | (0x9b7c >> 2),
390 (0x0e00 << 16) | (0x8a14 >> 2),
392 (0x0e00 << 16) | (0x8a18 >> 2),
394 (0x0600 << 16) | (0x30a00 >> 2),
396 (0x0e00 << 16) | (0x8bf0 >> 2),
398 (0x0e00 << 16) | (0x8bcc >> 2),
400 (0x0e00 << 16) | (0x8b24 >> 2),
402 (0x0e00 << 16) | (0x30a04 >> 2),
404 (0x0600 << 16) | (0x30a10 >> 2),
406 (0x0600 << 16) | (0x30a14 >> 2),
408 (0x0600 << 16) | (0x30a18 >> 2),
410 (0x0600 << 16) | (0x30a2c >> 2),
412 (0x0e00 << 16) | (0xc700 >> 2),
414 (0x0e00 << 16) | (0xc704 >> 2),
416 (0x0e00 << 16) | (0xc708 >> 2),
418 (0x0e00 << 16) | (0xc768 >> 2),
420 (0x0400 << 16) | (0xc770 >> 2),
422 (0x0400 << 16) | (0xc774 >> 2),
424 (0x0400 << 16) | (0xc778 >> 2),
426 (0x0400 << 16) | (0xc77c >> 2),
428 (0x0400 << 16) | (0xc780 >> 2),
430 (0x0400 << 16) | (0xc784 >> 2),
432 (0x0400 << 16) | (0xc788 >> 2),
434 (0x0400 << 16) | (0xc78c >> 2),
436 (0x0400 << 16) | (0xc798 >> 2),
438 (0x0400 << 16) | (0xc79c >> 2),
440 (0x0400 << 16) | (0xc7a0 >> 2),
442 (0x0400 << 16) | (0xc7a4 >> 2),
444 (0x0400 << 16) | (0xc7a8 >> 2),
446 (0x0400 << 16) | (0xc7ac >> 2),
448 (0x0400 << 16) | (0xc7b0 >> 2),
450 (0x0400 << 16) | (0xc7b4 >> 2),
452 (0x0e00 << 16) | (0x9100 >> 2),
454 (0x0e00 << 16) | (0x3c010 >> 2),
456 (0x0e00 << 16) | (0x92a8 >> 2),
458 (0x0e00 << 16) | (0x92ac >> 2),
460 (0x0e00 << 16) | (0x92b4 >> 2),
462 (0x0e00 << 16) | (0x92b8 >> 2),
464 (0x0e00 << 16) | (0x92bc >> 2),
466 (0x0e00 << 16) | (0x92c0 >> 2),
468 (0x0e00 << 16) | (0x92c4 >> 2),
470 (0x0e00 << 16) | (0x92c8 >> 2),
472 (0x0e00 << 16) | (0x92cc >> 2),
474 (0x0e00 << 16) | (0x92d0 >> 2),
476 (0x0e00 << 16) | (0x8c00 >> 2),
478 (0x0e00 << 16) | (0x8c04 >> 2),
480 (0x0e00 << 16) | (0x8c20 >> 2),
482 (0x0e00 << 16) | (0x8c38 >> 2),
484 (0x0e00 << 16) | (0x8c3c >> 2),
486 (0x0e00 << 16) | (0xae00 >> 2),
488 (0x0e00 << 16) | (0x9604 >> 2),
490 (0x0e00 << 16) | (0xac08 >> 2),
492 (0x0e00 << 16) | (0xac0c >> 2),
494 (0x0e00 << 16) | (0xac10 >> 2),
496 (0x0e00 << 16) | (0xac14 >> 2),
498 (0x0e00 << 16) | (0xac58 >> 2),
500 (0x0e00 << 16) | (0xac68 >> 2),
502 (0x0e00 << 16) | (0xac6c >> 2),
504 (0x0e00 << 16) | (0xac70 >> 2),
506 (0x0e00 << 16) | (0xac74 >> 2),
508 (0x0e00 << 16) | (0xac78 >> 2),
510 (0x0e00 << 16) | (0xac7c >> 2),
512 (0x0e00 << 16) | (0xac80 >> 2),
514 (0x0e00 << 16) | (0xac84 >> 2),
516 (0x0e00 << 16) | (0xac88 >> 2),
518 (0x0e00 << 16) | (0xac8c >> 2),
520 (0x0e00 << 16) | (0x970c >> 2),
522 (0x0e00 << 16) | (0x9714 >> 2),
524 (0x0e00 << 16) | (0x9718 >> 2),
526 (0x0e00 << 16) | (0x971c >> 2),
528 (0x0e00 << 16) | (0x31068 >> 2),
530 (0x4e00 << 16) | (0x31068 >> 2),
532 (0x5e00 << 16) | (0x31068 >> 2),
534 (0x6e00 << 16) | (0x31068 >> 2),
536 (0x7e00 << 16) | (0x31068 >> 2),
538 (0x8e00 << 16) | (0x31068 >> 2),
540 (0x9e00 << 16) | (0x31068 >> 2),
542 (0xae00 << 16) | (0x31068 >> 2),
544 (0xbe00 << 16) | (0x31068 >> 2),
546 (0x0e00 << 16) | (0xcd10 >> 2),
548 (0x0e00 << 16) | (0xcd14 >> 2),
550 (0x0e00 << 16) | (0x88b0 >> 2),
552 (0x0e00 << 16) | (0x88b4 >> 2),
554 (0x0e00 << 16) | (0x88b8 >> 2),
556 (0x0e00 << 16) | (0x88bc >> 2),
558 (0x0400 << 16) | (0x89c0 >> 2),
560 (0x0e00 << 16) | (0x88c4 >> 2),
562 (0x0e00 << 16) | (0x88c8 >> 2),
564 (0x0e00 << 16) | (0x88d0 >> 2),
566 (0x0e00 << 16) | (0x88d4 >> 2),
568 (0x0e00 << 16) | (0x88d8 >> 2),
570 (0x0e00 << 16) | (0x8980 >> 2),
572 (0x0e00 << 16) | (0x30938 >> 2),
574 (0x0e00 << 16) | (0x3093c >> 2),
576 (0x0e00 << 16) | (0x30940 >> 2),
578 (0x0e00 << 16) | (0x89a0 >> 2),
580 (0x0e00 << 16) | (0x30900 >> 2),
582 (0x0e00 << 16) | (0x30904 >> 2),
584 (0x0e00 << 16) | (0x89b4 >> 2),
586 (0x0e00 << 16) | (0x3c210 >> 2),
588 (0x0e00 << 16) | (0x3c214 >> 2),
590 (0x0e00 << 16) | (0x3c218 >> 2),
592 (0x0e00 << 16) | (0x8904 >> 2),
595 (0x0e00 << 16) | (0x8c28 >> 2),
596 (0x0e00 << 16) | (0x8c2c >> 2),
597 (0x0e00 << 16) | (0x8c30 >> 2),
598 (0x0e00 << 16) | (0x8c34 >> 2),
599 (0x0e00 << 16) | (0x9600 >> 2),
602 static const u32 kalindi_rlc_save_restore_register_list[] =
604 (0x0e00 << 16) | (0xc12c >> 2),
606 (0x0e00 << 16) | (0xc140 >> 2),
608 (0x0e00 << 16) | (0xc150 >> 2),
610 (0x0e00 << 16) | (0xc15c >> 2),
612 (0x0e00 << 16) | (0xc168 >> 2),
614 (0x0e00 << 16) | (0xc170 >> 2),
616 (0x0e00 << 16) | (0xc204 >> 2),
618 (0x0e00 << 16) | (0xc2b4 >> 2),
620 (0x0e00 << 16) | (0xc2b8 >> 2),
622 (0x0e00 << 16) | (0xc2bc >> 2),
624 (0x0e00 << 16) | (0xc2c0 >> 2),
626 (0x0e00 << 16) | (0x8228 >> 2),
628 (0x0e00 << 16) | (0x829c >> 2),
630 (0x0e00 << 16) | (0x869c >> 2),
632 (0x0600 << 16) | (0x98f4 >> 2),
634 (0x0e00 << 16) | (0x98f8 >> 2),
636 (0x0e00 << 16) | (0x9900 >> 2),
638 (0x0e00 << 16) | (0xc260 >> 2),
640 (0x0e00 << 16) | (0x90e8 >> 2),
642 (0x0e00 << 16) | (0x3c000 >> 2),
644 (0x0e00 << 16) | (0x3c00c >> 2),
646 (0x0e00 << 16) | (0x8c1c >> 2),
648 (0x0e00 << 16) | (0x9700 >> 2),
650 (0x0e00 << 16) | (0xcd20 >> 2),
652 (0x4e00 << 16) | (0xcd20 >> 2),
654 (0x5e00 << 16) | (0xcd20 >> 2),
656 (0x6e00 << 16) | (0xcd20 >> 2),
658 (0x7e00 << 16) | (0xcd20 >> 2),
660 (0x0e00 << 16) | (0x89bc >> 2),
662 (0x0e00 << 16) | (0x8900 >> 2),
665 (0x0e00 << 16) | (0xc130 >> 2),
667 (0x0e00 << 16) | (0xc134 >> 2),
669 (0x0e00 << 16) | (0xc1fc >> 2),
671 (0x0e00 << 16) | (0xc208 >> 2),
673 (0x0e00 << 16) | (0xc264 >> 2),
675 (0x0e00 << 16) | (0xc268 >> 2),
677 (0x0e00 << 16) | (0xc26c >> 2),
679 (0x0e00 << 16) | (0xc270 >> 2),
681 (0x0e00 << 16) | (0xc274 >> 2),
683 (0x0e00 << 16) | (0xc28c >> 2),
685 (0x0e00 << 16) | (0xc290 >> 2),
687 (0x0e00 << 16) | (0xc294 >> 2),
689 (0x0e00 << 16) | (0xc298 >> 2),
691 (0x0e00 << 16) | (0xc2a0 >> 2),
693 (0x0e00 << 16) | (0xc2a4 >> 2),
695 (0x0e00 << 16) | (0xc2a8 >> 2),
697 (0x0e00 << 16) | (0xc2ac >> 2),
699 (0x0e00 << 16) | (0x301d0 >> 2),
701 (0x0e00 << 16) | (0x30238 >> 2),
703 (0x0e00 << 16) | (0x30250 >> 2),
705 (0x0e00 << 16) | (0x30254 >> 2),
707 (0x0e00 << 16) | (0x30258 >> 2),
709 (0x0e00 << 16) | (0x3025c >> 2),
711 (0x4e00 << 16) | (0xc900 >> 2),
713 (0x5e00 << 16) | (0xc900 >> 2),
715 (0x6e00 << 16) | (0xc900 >> 2),
717 (0x7e00 << 16) | (0xc900 >> 2),
719 (0x4e00 << 16) | (0xc904 >> 2),
721 (0x5e00 << 16) | (0xc904 >> 2),
723 (0x6e00 << 16) | (0xc904 >> 2),
725 (0x7e00 << 16) | (0xc904 >> 2),
727 (0x4e00 << 16) | (0xc908 >> 2),
729 (0x5e00 << 16) | (0xc908 >> 2),
731 (0x6e00 << 16) | (0xc908 >> 2),
733 (0x7e00 << 16) | (0xc908 >> 2),
735 (0x4e00 << 16) | (0xc90c >> 2),
737 (0x5e00 << 16) | (0xc90c >> 2),
739 (0x6e00 << 16) | (0xc90c >> 2),
741 (0x7e00 << 16) | (0xc90c >> 2),
743 (0x4e00 << 16) | (0xc910 >> 2),
745 (0x5e00 << 16) | (0xc910 >> 2),
747 (0x6e00 << 16) | (0xc910 >> 2),
749 (0x7e00 << 16) | (0xc910 >> 2),
751 (0x0e00 << 16) | (0xc99c >> 2),
753 (0x0e00 << 16) | (0x9834 >> 2),
755 (0x0000 << 16) | (0x30f00 >> 2),
757 (0x0000 << 16) | (0x30f04 >> 2),
759 (0x0000 << 16) | (0x30f08 >> 2),
761 (0x0000 << 16) | (0x30f0c >> 2),
763 (0x0600 << 16) | (0x9b7c >> 2),
765 (0x0e00 << 16) | (0x8a14 >> 2),
767 (0x0e00 << 16) | (0x8a18 >> 2),
769 (0x0600 << 16) | (0x30a00 >> 2),
771 (0x0e00 << 16) | (0x8bf0 >> 2),
773 (0x0e00 << 16) | (0x8bcc >> 2),
775 (0x0e00 << 16) | (0x8b24 >> 2),
777 (0x0e00 << 16) | (0x30a04 >> 2),
779 (0x0600 << 16) | (0x30a10 >> 2),
781 (0x0600 << 16) | (0x30a14 >> 2),
783 (0x0600 << 16) | (0x30a18 >> 2),
785 (0x0600 << 16) | (0x30a2c >> 2),
787 (0x0e00 << 16) | (0xc700 >> 2),
789 (0x0e00 << 16) | (0xc704 >> 2),
791 (0x0e00 << 16) | (0xc708 >> 2),
793 (0x0e00 << 16) | (0xc768 >> 2),
795 (0x0400 << 16) | (0xc770 >> 2),
797 (0x0400 << 16) | (0xc774 >> 2),
799 (0x0400 << 16) | (0xc798 >> 2),
801 (0x0400 << 16) | (0xc79c >> 2),
803 (0x0e00 << 16) | (0x9100 >> 2),
805 (0x0e00 << 16) | (0x3c010 >> 2),
807 (0x0e00 << 16) | (0x8c00 >> 2),
809 (0x0e00 << 16) | (0x8c04 >> 2),
811 (0x0e00 << 16) | (0x8c20 >> 2),
813 (0x0e00 << 16) | (0x8c38 >> 2),
815 (0x0e00 << 16) | (0x8c3c >> 2),
817 (0x0e00 << 16) | (0xae00 >> 2),
819 (0x0e00 << 16) | (0x9604 >> 2),
821 (0x0e00 << 16) | (0xac08 >> 2),
823 (0x0e00 << 16) | (0xac0c >> 2),
825 (0x0e00 << 16) | (0xac10 >> 2),
827 (0x0e00 << 16) | (0xac14 >> 2),
829 (0x0e00 << 16) | (0xac58 >> 2),
831 (0x0e00 << 16) | (0xac68 >> 2),
833 (0x0e00 << 16) | (0xac6c >> 2),
835 (0x0e00 << 16) | (0xac70 >> 2),
837 (0x0e00 << 16) | (0xac74 >> 2),
839 (0x0e00 << 16) | (0xac78 >> 2),
841 (0x0e00 << 16) | (0xac7c >> 2),
843 (0x0e00 << 16) | (0xac80 >> 2),
845 (0x0e00 << 16) | (0xac84 >> 2),
847 (0x0e00 << 16) | (0xac88 >> 2),
849 (0x0e00 << 16) | (0xac8c >> 2),
851 (0x0e00 << 16) | (0x970c >> 2),
853 (0x0e00 << 16) | (0x9714 >> 2),
855 (0x0e00 << 16) | (0x9718 >> 2),
857 (0x0e00 << 16) | (0x971c >> 2),
859 (0x0e00 << 16) | (0x31068 >> 2),
861 (0x4e00 << 16) | (0x31068 >> 2),
863 (0x5e00 << 16) | (0x31068 >> 2),
865 (0x6e00 << 16) | (0x31068 >> 2),
867 (0x7e00 << 16) | (0x31068 >> 2),
869 (0x0e00 << 16) | (0xcd10 >> 2),
871 (0x0e00 << 16) | (0xcd14 >> 2),
873 (0x0e00 << 16) | (0x88b0 >> 2),
875 (0x0e00 << 16) | (0x88b4 >> 2),
877 (0x0e00 << 16) | (0x88b8 >> 2),
879 (0x0e00 << 16) | (0x88bc >> 2),
881 (0x0400 << 16) | (0x89c0 >> 2),
883 (0x0e00 << 16) | (0x88c4 >> 2),
885 (0x0e00 << 16) | (0x88c8 >> 2),
887 (0x0e00 << 16) | (0x88d0 >> 2),
889 (0x0e00 << 16) | (0x88d4 >> 2),
891 (0x0e00 << 16) | (0x88d8 >> 2),
893 (0x0e00 << 16) | (0x8980 >> 2),
895 (0x0e00 << 16) | (0x30938 >> 2),
897 (0x0e00 << 16) | (0x3093c >> 2),
899 (0x0e00 << 16) | (0x30940 >> 2),
901 (0x0e00 << 16) | (0x89a0 >> 2),
903 (0x0e00 << 16) | (0x30900 >> 2),
905 (0x0e00 << 16) | (0x30904 >> 2),
907 (0x0e00 << 16) | (0x89b4 >> 2),
909 (0x0e00 << 16) | (0x3e1fc >> 2),
911 (0x0e00 << 16) | (0x3c210 >> 2),
913 (0x0e00 << 16) | (0x3c214 >> 2),
915 (0x0e00 << 16) | (0x3c218 >> 2),
917 (0x0e00 << 16) | (0x8904 >> 2),
920 (0x0e00 << 16) | (0x8c28 >> 2),
921 (0x0e00 << 16) | (0x8c2c >> 2),
922 (0x0e00 << 16) | (0x8c30 >> 2),
923 (0x0e00 << 16) | (0x8c34 >> 2),
924 (0x0e00 << 16) | (0x9600 >> 2),
927 static const u32 bonaire_golden_spm_registers[] =
929 0x30800, 0xe0ffffff, 0xe0000000
932 static const u32 bonaire_golden_common_registers[] =
934 0xc770, 0xffffffff, 0x00000800,
935 0xc774, 0xffffffff, 0x00000800,
936 0xc798, 0xffffffff, 0x00007fbf,
937 0xc79c, 0xffffffff, 0x00007faf
940 static const u32 bonaire_golden_registers[] =
942 0x3354, 0x00000333, 0x00000333,
943 0x3350, 0x000c0fc0, 0x00040200,
944 0x9a10, 0x00010000, 0x00058208,
945 0x3c000, 0xffff1fff, 0x00140000,
946 0x3c200, 0xfdfc0fff, 0x00000100,
947 0x3c234, 0x40000000, 0x40000200,
948 0x9830, 0xffffffff, 0x00000000,
949 0x9834, 0xf00fffff, 0x00000400,
950 0x9838, 0x0002021c, 0x00020200,
951 0xc78, 0x00000080, 0x00000000,
952 0x5bb0, 0x000000f0, 0x00000070,
953 0x5bc0, 0xf0311fff, 0x80300000,
954 0x98f8, 0x73773777, 0x12010001,
955 0x350c, 0x00810000, 0x408af000,
956 0x7030, 0x31000111, 0x00000011,
957 0x2f48, 0x73773777, 0x12010001,
958 0x220c, 0x00007fb6, 0x0021a1b1,
959 0x2210, 0x00007fb6, 0x002021b1,
960 0x2180, 0x00007fb6, 0x00002191,
961 0x2218, 0x00007fb6, 0x002121b1,
962 0x221c, 0x00007fb6, 0x002021b1,
963 0x21dc, 0x00007fb6, 0x00002191,
964 0x21e0, 0x00007fb6, 0x00002191,
965 0x3628, 0x0000003f, 0x0000000a,
966 0x362c, 0x0000003f, 0x0000000a,
967 0x2ae4, 0x00073ffe, 0x000022a2,
968 0x240c, 0x000007ff, 0x00000000,
969 0x8a14, 0xf000003f, 0x00000007,
970 0x8bf0, 0x00002001, 0x00000001,
971 0x8b24, 0xffffffff, 0x00ffffff,
972 0x30a04, 0x0000ff0f, 0x00000000,
973 0x28a4c, 0x07ffffff, 0x06000000,
974 0x4d8, 0x00000fff, 0x00000100,
975 0x3e78, 0x00000001, 0x00000002,
976 0x9100, 0x03000000, 0x0362c688,
977 0x8c00, 0x000000ff, 0x00000001,
978 0xe40, 0x00001fff, 0x00001fff,
979 0x9060, 0x0000007f, 0x00000020,
980 0x9508, 0x00010000, 0x00010000,
981 0xac14, 0x000003ff, 0x000000f3,
982 0xac0c, 0xffffffff, 0x00001032
985 static const u32 bonaire_mgcg_cgcg_init[] =
987 0xc420, 0xffffffff, 0xfffffffc,
988 0x30800, 0xffffffff, 0xe0000000,
989 0x3c2a0, 0xffffffff, 0x00000100,
990 0x3c208, 0xffffffff, 0x00000100,
991 0x3c2c0, 0xffffffff, 0xc0000100,
992 0x3c2c8, 0xffffffff, 0xc0000100,
993 0x3c2c4, 0xffffffff, 0xc0000100,
994 0x55e4, 0xffffffff, 0x00600100,
995 0x3c280, 0xffffffff, 0x00000100,
996 0x3c214, 0xffffffff, 0x06000100,
997 0x3c220, 0xffffffff, 0x00000100,
998 0x3c218, 0xffffffff, 0x06000100,
999 0x3c204, 0xffffffff, 0x00000100,
1000 0x3c2e0, 0xffffffff, 0x00000100,
1001 0x3c224, 0xffffffff, 0x00000100,
1002 0x3c200, 0xffffffff, 0x00000100,
1003 0x3c230, 0xffffffff, 0x00000100,
1004 0x3c234, 0xffffffff, 0x00000100,
1005 0x3c250, 0xffffffff, 0x00000100,
1006 0x3c254, 0xffffffff, 0x00000100,
1007 0x3c258, 0xffffffff, 0x00000100,
1008 0x3c25c, 0xffffffff, 0x00000100,
1009 0x3c260, 0xffffffff, 0x00000100,
1010 0x3c27c, 0xffffffff, 0x00000100,
1011 0x3c278, 0xffffffff, 0x00000100,
1012 0x3c210, 0xffffffff, 0x06000100,
1013 0x3c290, 0xffffffff, 0x00000100,
1014 0x3c274, 0xffffffff, 0x00000100,
1015 0x3c2b4, 0xffffffff, 0x00000100,
1016 0x3c2b0, 0xffffffff, 0x00000100,
1017 0x3c270, 0xffffffff, 0x00000100,
1018 0x30800, 0xffffffff, 0xe0000000,
1019 0x3c020, 0xffffffff, 0x00010000,
1020 0x3c024, 0xffffffff, 0x00030002,
1021 0x3c028, 0xffffffff, 0x00040007,
1022 0x3c02c, 0xffffffff, 0x00060005,
1023 0x3c030, 0xffffffff, 0x00090008,
1024 0x3c034, 0xffffffff, 0x00010000,
1025 0x3c038, 0xffffffff, 0x00030002,
1026 0x3c03c, 0xffffffff, 0x00040007,
1027 0x3c040, 0xffffffff, 0x00060005,
1028 0x3c044, 0xffffffff, 0x00090008,
1029 0x3c048, 0xffffffff, 0x00010000,
1030 0x3c04c, 0xffffffff, 0x00030002,
1031 0x3c050, 0xffffffff, 0x00040007,
1032 0x3c054, 0xffffffff, 0x00060005,
1033 0x3c058, 0xffffffff, 0x00090008,
1034 0x3c05c, 0xffffffff, 0x00010000,
1035 0x3c060, 0xffffffff, 0x00030002,
1036 0x3c064, 0xffffffff, 0x00040007,
1037 0x3c068, 0xffffffff, 0x00060005,
1038 0x3c06c, 0xffffffff, 0x00090008,
1039 0x3c070, 0xffffffff, 0x00010000,
1040 0x3c074, 0xffffffff, 0x00030002,
1041 0x3c078, 0xffffffff, 0x00040007,
1042 0x3c07c, 0xffffffff, 0x00060005,
1043 0x3c080, 0xffffffff, 0x00090008,
1044 0x3c084, 0xffffffff, 0x00010000,
1045 0x3c088, 0xffffffff, 0x00030002,
1046 0x3c08c, 0xffffffff, 0x00040007,
1047 0x3c090, 0xffffffff, 0x00060005,
1048 0x3c094, 0xffffffff, 0x00090008,
1049 0x3c098, 0xffffffff, 0x00010000,
1050 0x3c09c, 0xffffffff, 0x00030002,
1051 0x3c0a0, 0xffffffff, 0x00040007,
1052 0x3c0a4, 0xffffffff, 0x00060005,
1053 0x3c0a8, 0xffffffff, 0x00090008,
1054 0x3c000, 0xffffffff, 0x96e00200,
1055 0x8708, 0xffffffff, 0x00900100,
1056 0xc424, 0xffffffff, 0x0020003f,
1057 0x38, 0xffffffff, 0x0140001c,
1058 0x3c, 0x000f0000, 0x000f0000,
1059 0x220, 0xffffffff, 0xC060000C,
1060 0x224, 0xc0000fff, 0x00000100,
1061 0xf90, 0xffffffff, 0x00000100,
1062 0xf98, 0x00000101, 0x00000000,
1063 0x20a8, 0xffffffff, 0x00000104,
1064 0x55e4, 0xff000fff, 0x00000100,
1065 0x30cc, 0xc0000fff, 0x00000104,
1066 0xc1e4, 0x00000001, 0x00000001,
1067 0xd00c, 0xff000ff0, 0x00000100,
1068 0xd80c, 0xff000ff0, 0x00000100
1071 static const u32 spectre_golden_spm_registers[] =
1073 0x30800, 0xe0ffffff, 0xe0000000
1076 static const u32 spectre_golden_common_registers[] =
1078 0xc770, 0xffffffff, 0x00000800,
1079 0xc774, 0xffffffff, 0x00000800,
1080 0xc798, 0xffffffff, 0x00007fbf,
1081 0xc79c, 0xffffffff, 0x00007faf
1084 static const u32 spectre_golden_registers[] =
1086 0x3c000, 0xffff1fff, 0x96940200,
1087 0x3c00c, 0xffff0001, 0xff000000,
1088 0x3c200, 0xfffc0fff, 0x00000100,
1089 0x6ed8, 0x00010101, 0x00010000,
1090 0x9834, 0xf00fffff, 0x00000400,
1091 0x9838, 0xfffffffc, 0x00020200,
1092 0x5bb0, 0x000000f0, 0x00000070,
1093 0x5bc0, 0xf0311fff, 0x80300000,
1094 0x98f8, 0x73773777, 0x12010001,
1095 0x9b7c, 0x00ff0000, 0x00fc0000,
1096 0x2f48, 0x73773777, 0x12010001,
1097 0x8a14, 0xf000003f, 0x00000007,
1098 0x8b24, 0xffffffff, 0x00ffffff,
1099 0x28350, 0x3f3f3fff, 0x00000082,
1100 0x28354, 0x0000003f, 0x00000000,
1101 0x3e78, 0x00000001, 0x00000002,
1102 0x913c, 0xffff03df, 0x00000004,
1103 0xc768, 0x00000008, 0x00000008,
1104 0x8c00, 0x000008ff, 0x00000800,
1105 0x9508, 0x00010000, 0x00010000,
1106 0xac0c, 0xffffffff, 0x54763210,
1107 0x214f8, 0x01ff01ff, 0x00000002,
1108 0x21498, 0x007ff800, 0x00200000,
1109 0x2015c, 0xffffffff, 0x00000f40,
1110 0x30934, 0xffffffff, 0x00000001
1113 static const u32 spectre_mgcg_cgcg_init[] =
1115 0xc420, 0xffffffff, 0xfffffffc,
1116 0x30800, 0xffffffff, 0xe0000000,
1117 0x3c2a0, 0xffffffff, 0x00000100,
1118 0x3c208, 0xffffffff, 0x00000100,
1119 0x3c2c0, 0xffffffff, 0x00000100,
1120 0x3c2c8, 0xffffffff, 0x00000100,
1121 0x3c2c4, 0xffffffff, 0x00000100,
1122 0x55e4, 0xffffffff, 0x00600100,
1123 0x3c280, 0xffffffff, 0x00000100,
1124 0x3c214, 0xffffffff, 0x06000100,
1125 0x3c220, 0xffffffff, 0x00000100,
1126 0x3c218, 0xffffffff, 0x06000100,
1127 0x3c204, 0xffffffff, 0x00000100,
1128 0x3c2e0, 0xffffffff, 0x00000100,
1129 0x3c224, 0xffffffff, 0x00000100,
1130 0x3c200, 0xffffffff, 0x00000100,
1131 0x3c230, 0xffffffff, 0x00000100,
1132 0x3c234, 0xffffffff, 0x00000100,
1133 0x3c250, 0xffffffff, 0x00000100,
1134 0x3c254, 0xffffffff, 0x00000100,
1135 0x3c258, 0xffffffff, 0x00000100,
1136 0x3c25c, 0xffffffff, 0x00000100,
1137 0x3c260, 0xffffffff, 0x00000100,
1138 0x3c27c, 0xffffffff, 0x00000100,
1139 0x3c278, 0xffffffff, 0x00000100,
1140 0x3c210, 0xffffffff, 0x06000100,
1141 0x3c290, 0xffffffff, 0x00000100,
1142 0x3c274, 0xffffffff, 0x00000100,
1143 0x3c2b4, 0xffffffff, 0x00000100,
1144 0x3c2b0, 0xffffffff, 0x00000100,
1145 0x3c270, 0xffffffff, 0x00000100,
1146 0x30800, 0xffffffff, 0xe0000000,
1147 0x3c020, 0xffffffff, 0x00010000,
1148 0x3c024, 0xffffffff, 0x00030002,
1149 0x3c028, 0xffffffff, 0x00040007,
1150 0x3c02c, 0xffffffff, 0x00060005,
1151 0x3c030, 0xffffffff, 0x00090008,
1152 0x3c034, 0xffffffff, 0x00010000,
1153 0x3c038, 0xffffffff, 0x00030002,
1154 0x3c03c, 0xffffffff, 0x00040007,
1155 0x3c040, 0xffffffff, 0x00060005,
1156 0x3c044, 0xffffffff, 0x00090008,
1157 0x3c048, 0xffffffff, 0x00010000,
1158 0x3c04c, 0xffffffff, 0x00030002,
1159 0x3c050, 0xffffffff, 0x00040007,
1160 0x3c054, 0xffffffff, 0x00060005,
1161 0x3c058, 0xffffffff, 0x00090008,
1162 0x3c05c, 0xffffffff, 0x00010000,
1163 0x3c060, 0xffffffff, 0x00030002,
1164 0x3c064, 0xffffffff, 0x00040007,
1165 0x3c068, 0xffffffff, 0x00060005,
1166 0x3c06c, 0xffffffff, 0x00090008,
1167 0x3c070, 0xffffffff, 0x00010000,
1168 0x3c074, 0xffffffff, 0x00030002,
1169 0x3c078, 0xffffffff, 0x00040007,
1170 0x3c07c, 0xffffffff, 0x00060005,
1171 0x3c080, 0xffffffff, 0x00090008,
1172 0x3c084, 0xffffffff, 0x00010000,
1173 0x3c088, 0xffffffff, 0x00030002,
1174 0x3c08c, 0xffffffff, 0x00040007,
1175 0x3c090, 0xffffffff, 0x00060005,
1176 0x3c094, 0xffffffff, 0x00090008,
1177 0x3c098, 0xffffffff, 0x00010000,
1178 0x3c09c, 0xffffffff, 0x00030002,
1179 0x3c0a0, 0xffffffff, 0x00040007,
1180 0x3c0a4, 0xffffffff, 0x00060005,
1181 0x3c0a8, 0xffffffff, 0x00090008,
1182 0x3c0ac, 0xffffffff, 0x00010000,
1183 0x3c0b0, 0xffffffff, 0x00030002,
1184 0x3c0b4, 0xffffffff, 0x00040007,
1185 0x3c0b8, 0xffffffff, 0x00060005,
1186 0x3c0bc, 0xffffffff, 0x00090008,
1187 0x3c000, 0xffffffff, 0x96e00200,
1188 0x8708, 0xffffffff, 0x00900100,
1189 0xc424, 0xffffffff, 0x0020003f,
1190 0x38, 0xffffffff, 0x0140001c,
1191 0x3c, 0x000f0000, 0x000f0000,
1192 0x220, 0xffffffff, 0xC060000C,
1193 0x224, 0xc0000fff, 0x00000100,
1194 0xf90, 0xffffffff, 0x00000100,
1195 0xf98, 0x00000101, 0x00000000,
1196 0x20a8, 0xffffffff, 0x00000104,
1197 0x55e4, 0xff000fff, 0x00000100,
1198 0x30cc, 0xc0000fff, 0x00000104,
1199 0xc1e4, 0x00000001, 0x00000001,
1200 0xd00c, 0xff000ff0, 0x00000100,
1201 0xd80c, 0xff000ff0, 0x00000100
1204 static const u32 kalindi_golden_spm_registers[] =
1206 0x30800, 0xe0ffffff, 0xe0000000
1209 static const u32 kalindi_golden_common_registers[] =
1211 0xc770, 0xffffffff, 0x00000800,
1212 0xc774, 0xffffffff, 0x00000800,
1213 0xc798, 0xffffffff, 0x00007fbf,
1214 0xc79c, 0xffffffff, 0x00007faf
1217 static const u32 kalindi_golden_registers[] =
1219 0x3c000, 0xffffdfff, 0x6e944040,
1220 0x55e4, 0xff607fff, 0xfc000100,
1221 0x3c220, 0xff000fff, 0x00000100,
1222 0x3c224, 0xff000fff, 0x00000100,
1223 0x3c200, 0xfffc0fff, 0x00000100,
1224 0x6ed8, 0x00010101, 0x00010000,
1225 0x9830, 0xffffffff, 0x00000000,
1226 0x9834, 0xf00fffff, 0x00000400,
1227 0x5bb0, 0x000000f0, 0x00000070,
1228 0x5bc0, 0xf0311fff, 0x80300000,
1229 0x98f8, 0x73773777, 0x12010001,
1230 0x98fc, 0xffffffff, 0x00000010,
1231 0x9b7c, 0x00ff0000, 0x00fc0000,
1232 0x8030, 0x00001f0f, 0x0000100a,
1233 0x2f48, 0x73773777, 0x12010001,
1234 0x2408, 0x000fffff, 0x000c007f,
1235 0x8a14, 0xf000003f, 0x00000007,
1236 0x8b24, 0x3fff3fff, 0x00ffcfff,
1237 0x30a04, 0x0000ff0f, 0x00000000,
1238 0x28a4c, 0x07ffffff, 0x06000000,
1239 0x4d8, 0x00000fff, 0x00000100,
1240 0x3e78, 0x00000001, 0x00000002,
1241 0xc768, 0x00000008, 0x00000008,
1242 0x8c00, 0x000000ff, 0x00000003,
1243 0x214f8, 0x01ff01ff, 0x00000002,
1244 0x21498, 0x007ff800, 0x00200000,
1245 0x2015c, 0xffffffff, 0x00000f40,
1246 0x88c4, 0x001f3ae3, 0x00000082,
1247 0x88d4, 0x0000001f, 0x00000010,
1248 0x30934, 0xffffffff, 0x00000000
1251 static const u32 kalindi_mgcg_cgcg_init[] =
1253 0xc420, 0xffffffff, 0xfffffffc,
1254 0x30800, 0xffffffff, 0xe0000000,
1255 0x3c2a0, 0xffffffff, 0x00000100,
1256 0x3c208, 0xffffffff, 0x00000100,
1257 0x3c2c0, 0xffffffff, 0x00000100,
1258 0x3c2c8, 0xffffffff, 0x00000100,
1259 0x3c2c4, 0xffffffff, 0x00000100,
1260 0x55e4, 0xffffffff, 0x00600100,
1261 0x3c280, 0xffffffff, 0x00000100,
1262 0x3c214, 0xffffffff, 0x06000100,
1263 0x3c220, 0xffffffff, 0x00000100,
1264 0x3c218, 0xffffffff, 0x06000100,
1265 0x3c204, 0xffffffff, 0x00000100,
1266 0x3c2e0, 0xffffffff, 0x00000100,
1267 0x3c224, 0xffffffff, 0x00000100,
1268 0x3c200, 0xffffffff, 0x00000100,
1269 0x3c230, 0xffffffff, 0x00000100,
1270 0x3c234, 0xffffffff, 0x00000100,
1271 0x3c250, 0xffffffff, 0x00000100,
1272 0x3c254, 0xffffffff, 0x00000100,
1273 0x3c258, 0xffffffff, 0x00000100,
1274 0x3c25c, 0xffffffff, 0x00000100,
1275 0x3c260, 0xffffffff, 0x00000100,
1276 0x3c27c, 0xffffffff, 0x00000100,
1277 0x3c278, 0xffffffff, 0x00000100,
1278 0x3c210, 0xffffffff, 0x06000100,
1279 0x3c290, 0xffffffff, 0x00000100,
1280 0x3c274, 0xffffffff, 0x00000100,
1281 0x3c2b4, 0xffffffff, 0x00000100,
1282 0x3c2b0, 0xffffffff, 0x00000100,
1283 0x3c270, 0xffffffff, 0x00000100,
1284 0x30800, 0xffffffff, 0xe0000000,
1285 0x3c020, 0xffffffff, 0x00010000,
1286 0x3c024, 0xffffffff, 0x00030002,
1287 0x3c028, 0xffffffff, 0x00040007,
1288 0x3c02c, 0xffffffff, 0x00060005,
1289 0x3c030, 0xffffffff, 0x00090008,
1290 0x3c034, 0xffffffff, 0x00010000,
1291 0x3c038, 0xffffffff, 0x00030002,
1292 0x3c03c, 0xffffffff, 0x00040007,
1293 0x3c040, 0xffffffff, 0x00060005,
1294 0x3c044, 0xffffffff, 0x00090008,
1295 0x3c000, 0xffffffff, 0x96e00200,
1296 0x8708, 0xffffffff, 0x00900100,
1297 0xc424, 0xffffffff, 0x0020003f,
1298 0x38, 0xffffffff, 0x0140001c,
1299 0x3c, 0x000f0000, 0x000f0000,
1300 0x220, 0xffffffff, 0xC060000C,
1301 0x224, 0xc0000fff, 0x00000100,
1302 0x20a8, 0xffffffff, 0x00000104,
1303 0x55e4, 0xff000fff, 0x00000100,
1304 0x30cc, 0xc0000fff, 0x00000104,
1305 0xc1e4, 0x00000001, 0x00000001,
1306 0xd00c, 0xff000ff0, 0x00000100,
1307 0xd80c, 0xff000ff0, 0x00000100
1310 static const u32 hawaii_golden_spm_registers[] =
1312 0x30800, 0xe0ffffff, 0xe0000000
1315 static const u32 hawaii_golden_common_registers[] =
1317 0x30800, 0xffffffff, 0xe0000000,
1318 0x28350, 0xffffffff, 0x3a00161a,
1319 0x28354, 0xffffffff, 0x0000002e,
1320 0x9a10, 0xffffffff, 0x00018208,
1321 0x98f8, 0xffffffff, 0x12011003
1324 static const u32 hawaii_golden_registers[] =
1326 0x3354, 0x00000333, 0x00000333,
1327 0x9a10, 0x00010000, 0x00058208,
1328 0x9830, 0xffffffff, 0x00000000,
1329 0x9834, 0xf00fffff, 0x00000400,
1330 0x9838, 0x0002021c, 0x00020200,
1331 0xc78, 0x00000080, 0x00000000,
1332 0x5bb0, 0x000000f0, 0x00000070,
1333 0x5bc0, 0xf0311fff, 0x80300000,
1334 0x350c, 0x00810000, 0x408af000,
1335 0x7030, 0x31000111, 0x00000011,
1336 0x2f48, 0x73773777, 0x12010001,
1337 0x2120, 0x0000007f, 0x0000001b,
1338 0x21dc, 0x00007fb6, 0x00002191,
1339 0x3628, 0x0000003f, 0x0000000a,
1340 0x362c, 0x0000003f, 0x0000000a,
1341 0x2ae4, 0x00073ffe, 0x000022a2,
1342 0x240c, 0x000007ff, 0x00000000,
1343 0x8bf0, 0x00002001, 0x00000001,
1344 0x8b24, 0xffffffff, 0x00ffffff,
1345 0x30a04, 0x0000ff0f, 0x00000000,
1346 0x28a4c, 0x07ffffff, 0x06000000,
1347 0x3e78, 0x00000001, 0x00000002,
1348 0xc768, 0x00000008, 0x00000008,
1349 0xc770, 0x00000f00, 0x00000800,
1350 0xc774, 0x00000f00, 0x00000800,
1351 0xc798, 0x00ffffff, 0x00ff7fbf,
1352 0xc79c, 0x00ffffff, 0x00ff7faf,
1353 0x8c00, 0x000000ff, 0x00000800,
1354 0xe40, 0x00001fff, 0x00001fff,
1355 0x9060, 0x0000007f, 0x00000020,
1356 0x9508, 0x00010000, 0x00010000,
1357 0xae00, 0x00100000, 0x000ff07c,
1358 0xac14, 0x000003ff, 0x0000000f,
1359 0xac10, 0xffffffff, 0x7564fdec,
1360 0xac0c, 0xffffffff, 0x3120b9a8,
1361 0xac08, 0x20000000, 0x0f9c0000
1364 static const u32 hawaii_mgcg_cgcg_init[] =
1366 0xc420, 0xffffffff, 0xfffffffd,
1367 0x30800, 0xffffffff, 0xe0000000,
1368 0x3c2a0, 0xffffffff, 0x00000100,
1369 0x3c208, 0xffffffff, 0x00000100,
1370 0x3c2c0, 0xffffffff, 0x00000100,
1371 0x3c2c8, 0xffffffff, 0x00000100,
1372 0x3c2c4, 0xffffffff, 0x00000100,
1373 0x55e4, 0xffffffff, 0x00200100,
1374 0x3c280, 0xffffffff, 0x00000100,
1375 0x3c214, 0xffffffff, 0x06000100,
1376 0x3c220, 0xffffffff, 0x00000100,
1377 0x3c218, 0xffffffff, 0x06000100,
1378 0x3c204, 0xffffffff, 0x00000100,
1379 0x3c2e0, 0xffffffff, 0x00000100,
1380 0x3c224, 0xffffffff, 0x00000100,
1381 0x3c200, 0xffffffff, 0x00000100,
1382 0x3c230, 0xffffffff, 0x00000100,
1383 0x3c234, 0xffffffff, 0x00000100,
1384 0x3c250, 0xffffffff, 0x00000100,
1385 0x3c254, 0xffffffff, 0x00000100,
1386 0x3c258, 0xffffffff, 0x00000100,
1387 0x3c25c, 0xffffffff, 0x00000100,
1388 0x3c260, 0xffffffff, 0x00000100,
1389 0x3c27c, 0xffffffff, 0x00000100,
1390 0x3c278, 0xffffffff, 0x00000100,
1391 0x3c210, 0xffffffff, 0x06000100,
1392 0x3c290, 0xffffffff, 0x00000100,
1393 0x3c274, 0xffffffff, 0x00000100,
1394 0x3c2b4, 0xffffffff, 0x00000100,
1395 0x3c2b0, 0xffffffff, 0x00000100,
1396 0x3c270, 0xffffffff, 0x00000100,
1397 0x30800, 0xffffffff, 0xe0000000,
1398 0x3c020, 0xffffffff, 0x00010000,
1399 0x3c024, 0xffffffff, 0x00030002,
1400 0x3c028, 0xffffffff, 0x00040007,
1401 0x3c02c, 0xffffffff, 0x00060005,
1402 0x3c030, 0xffffffff, 0x00090008,
1403 0x3c034, 0xffffffff, 0x00010000,
1404 0x3c038, 0xffffffff, 0x00030002,
1405 0x3c03c, 0xffffffff, 0x00040007,
1406 0x3c040, 0xffffffff, 0x00060005,
1407 0x3c044, 0xffffffff, 0x00090008,
1408 0x3c048, 0xffffffff, 0x00010000,
1409 0x3c04c, 0xffffffff, 0x00030002,
1410 0x3c050, 0xffffffff, 0x00040007,
1411 0x3c054, 0xffffffff, 0x00060005,
1412 0x3c058, 0xffffffff, 0x00090008,
1413 0x3c05c, 0xffffffff, 0x00010000,
1414 0x3c060, 0xffffffff, 0x00030002,
1415 0x3c064, 0xffffffff, 0x00040007,
1416 0x3c068, 0xffffffff, 0x00060005,
1417 0x3c06c, 0xffffffff, 0x00090008,
1418 0x3c070, 0xffffffff, 0x00010000,
1419 0x3c074, 0xffffffff, 0x00030002,
1420 0x3c078, 0xffffffff, 0x00040007,
1421 0x3c07c, 0xffffffff, 0x00060005,
1422 0x3c080, 0xffffffff, 0x00090008,
1423 0x3c084, 0xffffffff, 0x00010000,
1424 0x3c088, 0xffffffff, 0x00030002,
1425 0x3c08c, 0xffffffff, 0x00040007,
1426 0x3c090, 0xffffffff, 0x00060005,
1427 0x3c094, 0xffffffff, 0x00090008,
1428 0x3c098, 0xffffffff, 0x00010000,
1429 0x3c09c, 0xffffffff, 0x00030002,
1430 0x3c0a0, 0xffffffff, 0x00040007,
1431 0x3c0a4, 0xffffffff, 0x00060005,
1432 0x3c0a8, 0xffffffff, 0x00090008,
1433 0x3c0ac, 0xffffffff, 0x00010000,
1434 0x3c0b0, 0xffffffff, 0x00030002,
1435 0x3c0b4, 0xffffffff, 0x00040007,
1436 0x3c0b8, 0xffffffff, 0x00060005,
1437 0x3c0bc, 0xffffffff, 0x00090008,
1438 0x3c0c0, 0xffffffff, 0x00010000,
1439 0x3c0c4, 0xffffffff, 0x00030002,
1440 0x3c0c8, 0xffffffff, 0x00040007,
1441 0x3c0cc, 0xffffffff, 0x00060005,
1442 0x3c0d0, 0xffffffff, 0x00090008,
1443 0x3c0d4, 0xffffffff, 0x00010000,
1444 0x3c0d8, 0xffffffff, 0x00030002,
1445 0x3c0dc, 0xffffffff, 0x00040007,
1446 0x3c0e0, 0xffffffff, 0x00060005,
1447 0x3c0e4, 0xffffffff, 0x00090008,
1448 0x3c0e8, 0xffffffff, 0x00010000,
1449 0x3c0ec, 0xffffffff, 0x00030002,
1450 0x3c0f0, 0xffffffff, 0x00040007,
1451 0x3c0f4, 0xffffffff, 0x00060005,
1452 0x3c0f8, 0xffffffff, 0x00090008,
1453 0xc318, 0xffffffff, 0x00020200,
1454 0x3350, 0xffffffff, 0x00000200,
1455 0x15c0, 0xffffffff, 0x00000400,
1456 0x55e8, 0xffffffff, 0x00000000,
1457 0x2f50, 0xffffffff, 0x00000902,
1458 0x3c000, 0xffffffff, 0x96940200,
1459 0x8708, 0xffffffff, 0x00900100,
1460 0xc424, 0xffffffff, 0x0020003f,
1461 0x38, 0xffffffff, 0x0140001c,
1462 0x3c, 0x000f0000, 0x000f0000,
1463 0x220, 0xffffffff, 0xc060000c,
1464 0x224, 0xc0000fff, 0x00000100,
1465 0xf90, 0xffffffff, 0x00000100,
1466 0xf98, 0x00000101, 0x00000000,
1467 0x20a8, 0xffffffff, 0x00000104,
1468 0x55e4, 0xff000fff, 0x00000100,
1469 0x30cc, 0xc0000fff, 0x00000104,
1470 0xc1e4, 0x00000001, 0x00000001,
1471 0xd00c, 0xff000ff0, 0x00000100,
1472 0xd80c, 0xff000ff0, 0x00000100
1475 static void cik_init_golden_registers(struct radeon_device *rdev)
1477 switch (rdev->family) {
1479 radeon_program_register_sequence(rdev,
1480 bonaire_mgcg_cgcg_init,
1481 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
1482 radeon_program_register_sequence(rdev,
1483 bonaire_golden_registers,
1484 (const u32)ARRAY_SIZE(bonaire_golden_registers));
1485 radeon_program_register_sequence(rdev,
1486 bonaire_golden_common_registers,
1487 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
1488 radeon_program_register_sequence(rdev,
1489 bonaire_golden_spm_registers,
1490 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
1493 radeon_program_register_sequence(rdev,
1494 kalindi_mgcg_cgcg_init,
1495 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
1496 radeon_program_register_sequence(rdev,
1497 kalindi_golden_registers,
1498 (const u32)ARRAY_SIZE(kalindi_golden_registers));
1499 radeon_program_register_sequence(rdev,
1500 kalindi_golden_common_registers,
1501 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
1502 radeon_program_register_sequence(rdev,
1503 kalindi_golden_spm_registers,
1504 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
1507 radeon_program_register_sequence(rdev,
1508 spectre_mgcg_cgcg_init,
1509 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
1510 radeon_program_register_sequence(rdev,
1511 spectre_golden_registers,
1512 (const u32)ARRAY_SIZE(spectre_golden_registers));
1513 radeon_program_register_sequence(rdev,
1514 spectre_golden_common_registers,
1515 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
1516 radeon_program_register_sequence(rdev,
1517 spectre_golden_spm_registers,
1518 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
1521 radeon_program_register_sequence(rdev,
1522 hawaii_mgcg_cgcg_init,
1523 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
1524 radeon_program_register_sequence(rdev,
1525 hawaii_golden_registers,
1526 (const u32)ARRAY_SIZE(hawaii_golden_registers));
1527 radeon_program_register_sequence(rdev,
1528 hawaii_golden_common_registers,
1529 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
1530 radeon_program_register_sequence(rdev,
1531 hawaii_golden_spm_registers,
1532 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
1540 * cik_get_xclk - get the xclk
1542 * @rdev: radeon_device pointer
1544 * Returns the reference clock used by the gfx engine
1547 u32 cik_get_xclk(struct radeon_device *rdev)
1549 u32 reference_clock = rdev->clock.spll.reference_freq;
1551 if (rdev->flags & RADEON_IS_IGP) {
1552 if (RREG32_SMC(GENERAL_PWRMGT) & GPU_COUNTER_CLK)
1553 return reference_clock / 2;
1555 if (RREG32_SMC(CG_CLKPIN_CNTL) & XTALIN_DIVIDE)
1556 return reference_clock / 4;
1558 return reference_clock;
1562 * cik_mm_rdoorbell - read a doorbell dword
1564 * @rdev: radeon_device pointer
1565 * @index: doorbell index
1567 * Returns the value in the doorbell aperture at the
1568 * requested doorbell index (CIK).
1570 u32 cik_mm_rdoorbell(struct radeon_device *rdev, u32 index)
1572 if (index < rdev->doorbell.num_doorbells) {
1573 return readl(rdev->doorbell.ptr + index);
1575 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
1581 * cik_mm_wdoorbell - write a doorbell dword
1583 * @rdev: radeon_device pointer
1584 * @index: doorbell index
1585 * @v: value to write
1587 * Writes @v to the doorbell aperture at the
1588 * requested doorbell index (CIK).
1590 void cik_mm_wdoorbell(struct radeon_device *rdev, u32 index, u32 v)
1592 if (index < rdev->doorbell.num_doorbells) {
1593 writel(v, rdev->doorbell.ptr + index);
1595 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
1599 #define BONAIRE_IO_MC_REGS_SIZE 36
1601 static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
1603 {0x00000070, 0x04400000},
1604 {0x00000071, 0x80c01803},
1605 {0x00000072, 0x00004004},
1606 {0x00000073, 0x00000100},
1607 {0x00000074, 0x00ff0000},
1608 {0x00000075, 0x34000000},
1609 {0x00000076, 0x08000014},
1610 {0x00000077, 0x00cc08ec},
1611 {0x00000078, 0x00000400},
1612 {0x00000079, 0x00000000},
1613 {0x0000007a, 0x04090000},
1614 {0x0000007c, 0x00000000},
1615 {0x0000007e, 0x4408a8e8},
1616 {0x0000007f, 0x00000304},
1617 {0x00000080, 0x00000000},
1618 {0x00000082, 0x00000001},
1619 {0x00000083, 0x00000002},
1620 {0x00000084, 0xf3e4f400},
1621 {0x00000085, 0x052024e3},
1622 {0x00000087, 0x00000000},
1623 {0x00000088, 0x01000000},
1624 {0x0000008a, 0x1c0a0000},
1625 {0x0000008b, 0xff010000},
1626 {0x0000008d, 0xffffefff},
1627 {0x0000008e, 0xfff3efff},
1628 {0x0000008f, 0xfff3efbf},
1629 {0x00000092, 0xf7ffffff},
1630 {0x00000093, 0xffffff7f},
1631 {0x00000095, 0x00101101},
1632 {0x00000096, 0x00000fff},
1633 {0x00000097, 0x00116fff},
1634 {0x00000098, 0x60010000},
1635 {0x00000099, 0x10010000},
1636 {0x0000009a, 0x00006000},
1637 {0x0000009b, 0x00001000},
1638 {0x0000009f, 0x00b48000}
1641 #define HAWAII_IO_MC_REGS_SIZE 22
1643 static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
1645 {0x0000007d, 0x40000000},
1646 {0x0000007e, 0x40180304},
1647 {0x0000007f, 0x0000ff00},
1648 {0x00000081, 0x00000000},
1649 {0x00000083, 0x00000800},
1650 {0x00000086, 0x00000000},
1651 {0x00000087, 0x00000100},
1652 {0x00000088, 0x00020100},
1653 {0x00000089, 0x00000000},
1654 {0x0000008b, 0x00040000},
1655 {0x0000008c, 0x00000100},
1656 {0x0000008e, 0xff010000},
1657 {0x00000090, 0xffffefff},
1658 {0x00000091, 0xfff3efff},
1659 {0x00000092, 0xfff3efbf},
1660 {0x00000093, 0xf7ffffff},
1661 {0x00000094, 0xffffff7f},
1662 {0x00000095, 0x00000fff},
1663 {0x00000096, 0x00116fff},
1664 {0x00000097, 0x60010000},
1665 {0x00000098, 0x10010000},
1666 {0x0000009f, 0x00c79000}
1671 * cik_srbm_select - select specific register instances
1673 * @rdev: radeon_device pointer
1674 * @me: selected ME (micro engine)
1679 * Switches the currently active registers instances. Some
1680 * registers are instanced per VMID, others are instanced per
1681 * me/pipe/queue combination.
1683 static void cik_srbm_select(struct radeon_device *rdev,
1684 u32 me, u32 pipe, u32 queue, u32 vmid)
1686 u32 srbm_gfx_cntl = (PIPEID(pipe & 0x3) |
1689 QUEUEID(queue & 0x7));
1690 WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl);
1695 * ci_mc_load_microcode - load MC ucode into the hw
1697 * @rdev: radeon_device pointer
1699 * Load the GDDR MC ucode into the hw (CIK).
1700 * Returns 0 on success, error on failure.
1702 int ci_mc_load_microcode(struct radeon_device *rdev)
1704 const __be32 *fw_data;
1705 u32 running, blackout = 0;
1707 int i, regs_size, ucode_size;
1712 ucode_size = rdev->mc_fw->size / 4;
1714 switch (rdev->family) {
1716 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
1717 regs_size = BONAIRE_IO_MC_REGS_SIZE;
1720 io_mc_regs = (u32 *)&hawaii_io_mc_regs;
1721 regs_size = HAWAII_IO_MC_REGS_SIZE;
1727 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1731 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1732 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1735 /* reset the engine and set to writable */
1736 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1737 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1739 /* load mc io regs */
1740 for (i = 0; i < regs_size; i++) {
1741 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1742 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1744 /* load the MC ucode */
1745 fw_data = (const __be32 *)rdev->mc_fw->data;
1746 for (i = 0; i < ucode_size; i++)
1747 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1749 /* put the engine back into the active state */
1750 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1751 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1752 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1754 /* wait for training to complete */
1755 for (i = 0; i < rdev->usec_timeout; i++) {
1756 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1760 for (i = 0; i < rdev->usec_timeout; i++) {
1761 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1767 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1774 * cik_init_microcode - load ucode images from disk
1776 * @rdev: radeon_device pointer
1778 * Use the firmware interface to load the ucode images into
1779 * the driver (not loaded into hw).
1780 * Returns 0 on success, error on failure.
1782 static int cik_init_microcode(struct radeon_device *rdev)
1784 const char *chip_name;
1785 size_t pfp_req_size, me_req_size, ce_req_size,
1786 mec_req_size, rlc_req_size, mc_req_size = 0,
1787 sdma_req_size, smc_req_size = 0, mc2_req_size = 0;
1793 switch (rdev->family) {
1795 chip_name = "BONAIRE";
1796 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1797 me_req_size = CIK_ME_UCODE_SIZE * 4;
1798 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1799 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1800 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1801 mc_req_size = BONAIRE_MC_UCODE_SIZE * 4;
1802 mc2_req_size = BONAIRE_MC2_UCODE_SIZE * 4;
1803 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1804 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
1807 chip_name = "HAWAII";
1808 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1809 me_req_size = CIK_ME_UCODE_SIZE * 4;
1810 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1811 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1812 rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
1813 mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
1814 mc2_req_size = HAWAII_MC2_UCODE_SIZE * 4;
1815 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1816 smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
1819 chip_name = "KAVERI";
1820 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1821 me_req_size = CIK_ME_UCODE_SIZE * 4;
1822 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1823 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1824 rlc_req_size = KV_RLC_UCODE_SIZE * 4;
1825 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1828 chip_name = "KABINI";
1829 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
1830 me_req_size = CIK_ME_UCODE_SIZE * 4;
1831 ce_req_size = CIK_CE_UCODE_SIZE * 4;
1832 mec_req_size = CIK_MEC_UCODE_SIZE * 4;
1833 rlc_req_size = KB_RLC_UCODE_SIZE * 4;
1834 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
1839 DRM_INFO("Loading %s Microcode\n", chip_name);
1841 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1842 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1845 if (rdev->pfp_fw->size != pfp_req_size) {
1847 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1848 rdev->pfp_fw->size, fw_name);
1853 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1854 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1857 if (rdev->me_fw->size != me_req_size) {
1859 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1860 rdev->me_fw->size, fw_name);
1864 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1865 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1868 if (rdev->ce_fw->size != ce_req_size) {
1870 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1871 rdev->ce_fw->size, fw_name);
1875 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mec.bin", chip_name);
1876 err = request_firmware(&rdev->mec_fw, fw_name, rdev->dev);
1879 if (rdev->mec_fw->size != mec_req_size) {
1881 "cik_cp: Bogus length %zu in firmware \"%s\"\n",
1882 rdev->mec_fw->size, fw_name);
1886 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
1887 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1890 if (rdev->rlc_fw->size != rlc_req_size) {
1892 "cik_rlc: Bogus length %zu in firmware \"%s\"\n",
1893 rdev->rlc_fw->size, fw_name);
1897 snprintf(fw_name, sizeof(fw_name), "radeon/%s_sdma.bin", chip_name);
1898 err = request_firmware(&rdev->sdma_fw, fw_name, rdev->dev);
1901 if (rdev->sdma_fw->size != sdma_req_size) {
1903 "cik_sdma: Bogus length %zu in firmware \"%s\"\n",
1904 rdev->sdma_fw->size, fw_name);
1908 /* No SMC, MC ucode on APUs */
1909 if (!(rdev->flags & RADEON_IS_IGP)) {
1910 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
1911 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1913 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1914 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1918 if ((rdev->mc_fw->size != mc_req_size) &&
1919 (rdev->mc_fw->size != mc2_req_size)){
1921 "cik_mc: Bogus length %zu in firmware \"%s\"\n",
1922 rdev->mc_fw->size, fw_name);
1925 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
1927 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1928 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1931 "smc: error loading firmware \"%s\"\n",
1933 release_firmware(rdev->smc_fw);
1934 rdev->smc_fw = NULL;
1936 } else if (rdev->smc_fw->size != smc_req_size) {
1938 "cik_smc: Bogus length %zu in firmware \"%s\"\n",
1939 rdev->smc_fw->size, fw_name);
1948 "cik_cp: Failed to load firmware \"%s\"\n",
1950 release_firmware(rdev->pfp_fw);
1951 rdev->pfp_fw = NULL;
1952 release_firmware(rdev->me_fw);
1954 release_firmware(rdev->ce_fw);
1956 release_firmware(rdev->rlc_fw);
1957 rdev->rlc_fw = NULL;
1958 release_firmware(rdev->mc_fw);
1960 release_firmware(rdev->smc_fw);
1961 rdev->smc_fw = NULL;
1970 * cik_tiling_mode_table_init - init the hw tiling table
1972 * @rdev: radeon_device pointer
1974 * Starting with SI, the tiling setup is done globally in a
1975 * set of 32 tiling modes. Rather than selecting each set of
1976 * parameters per surface as on older asics, we just select
1977 * which index in the tiling table we want to use, and the
1978 * surface uses those parameters (CIK).
1980 static void cik_tiling_mode_table_init(struct radeon_device *rdev)
1982 const u32 num_tile_mode_states = 32;
1983 const u32 num_secondary_tile_mode_states = 16;
1984 u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
1985 u32 num_pipe_configs;
1986 u32 num_rbs = rdev->config.cik.max_backends_per_se *
1987 rdev->config.cik.max_shader_engines;
1989 switch (rdev->config.cik.mem_row_size_in_kb) {
1991 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
1995 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
1998 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2002 num_pipe_configs = rdev->config.cik.max_tile_pipes;
2003 if (num_pipe_configs > 8)
2004 num_pipe_configs = 16;
2006 if (num_pipe_configs == 16) {
2007 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2008 switch (reg_offset) {
2010 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2011 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2012 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2013 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2016 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2017 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2018 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2019 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2022 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2023 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2024 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2025 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2028 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2029 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2030 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2031 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2034 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2035 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2036 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2037 TILE_SPLIT(split_equal_to_row_size));
2040 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2041 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2044 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2045 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2046 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2047 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2050 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2051 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2052 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2053 TILE_SPLIT(split_equal_to_row_size));
2056 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2057 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
2060 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2061 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2064 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2065 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2066 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2067 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2070 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2071 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2072 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2073 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2076 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2077 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2078 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2079 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2082 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2083 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2086 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2087 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2088 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2089 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2092 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2093 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2094 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2095 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2098 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2099 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2100 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2101 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2104 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2105 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2108 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2109 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2110 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2111 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2114 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2115 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2116 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
2117 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2120 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2121 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2122 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
2123 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2129 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2130 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2132 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2133 switch (reg_offset) {
2135 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2136 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2137 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2138 NUM_BANKS(ADDR_SURF_16_BANK));
2141 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2142 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2143 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2144 NUM_BANKS(ADDR_SURF_16_BANK));
2147 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2148 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2149 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2150 NUM_BANKS(ADDR_SURF_16_BANK));
2153 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2154 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2155 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2156 NUM_BANKS(ADDR_SURF_16_BANK));
2159 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2160 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2161 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2162 NUM_BANKS(ADDR_SURF_8_BANK));
2165 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2166 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2167 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2168 NUM_BANKS(ADDR_SURF_4_BANK));
2171 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2172 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2173 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2174 NUM_BANKS(ADDR_SURF_2_BANK));
2177 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2178 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2179 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2180 NUM_BANKS(ADDR_SURF_16_BANK));
2183 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2184 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2185 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2186 NUM_BANKS(ADDR_SURF_16_BANK));
2189 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2190 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2191 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2192 NUM_BANKS(ADDR_SURF_16_BANK));
2195 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2196 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2197 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2198 NUM_BANKS(ADDR_SURF_8_BANK));
2201 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2202 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2203 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2204 NUM_BANKS(ADDR_SURF_4_BANK));
2207 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2208 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2209 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2210 NUM_BANKS(ADDR_SURF_2_BANK));
2213 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2214 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2215 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2216 NUM_BANKS(ADDR_SURF_2_BANK));
2222 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2223 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2225 } else if (num_pipe_configs == 8) {
2226 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2227 switch (reg_offset) {
2229 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2230 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2231 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2232 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2235 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2236 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2237 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2238 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2241 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2242 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2243 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2244 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2247 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2248 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2249 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2250 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2253 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2254 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2255 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2256 TILE_SPLIT(split_equal_to_row_size));
2259 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2260 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2263 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2264 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2265 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2266 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2269 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2270 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2271 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2272 TILE_SPLIT(split_equal_to_row_size));
2275 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2276 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16));
2279 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2280 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2283 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2284 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2285 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2286 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2289 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2290 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2291 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2292 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2295 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2296 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2297 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2298 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2301 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2302 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2305 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2306 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2307 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2308 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2311 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2312 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2313 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2314 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2317 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2318 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2319 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2320 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2323 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2324 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2327 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2328 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2329 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2330 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2333 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2334 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2335 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2336 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2339 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2340 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2341 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
2342 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2348 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2349 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2351 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2352 switch (reg_offset) {
2354 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2355 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2356 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2357 NUM_BANKS(ADDR_SURF_16_BANK));
2360 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2361 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2362 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2363 NUM_BANKS(ADDR_SURF_16_BANK));
2366 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2367 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2368 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2369 NUM_BANKS(ADDR_SURF_16_BANK));
2372 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2373 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2374 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2375 NUM_BANKS(ADDR_SURF_16_BANK));
2378 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2379 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2380 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2381 NUM_BANKS(ADDR_SURF_8_BANK));
2384 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2385 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2386 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2387 NUM_BANKS(ADDR_SURF_4_BANK));
2390 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2391 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2392 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2393 NUM_BANKS(ADDR_SURF_2_BANK));
2396 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2397 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2398 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2399 NUM_BANKS(ADDR_SURF_16_BANK));
2402 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2403 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2404 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2405 NUM_BANKS(ADDR_SURF_16_BANK));
2408 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2409 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2410 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2411 NUM_BANKS(ADDR_SURF_16_BANK));
2414 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2415 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2416 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2417 NUM_BANKS(ADDR_SURF_16_BANK));
2420 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2421 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2422 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2423 NUM_BANKS(ADDR_SURF_8_BANK));
2426 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2427 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2428 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2429 NUM_BANKS(ADDR_SURF_4_BANK));
2432 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2433 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2434 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2435 NUM_BANKS(ADDR_SURF_2_BANK));
2441 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2442 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2444 } else if (num_pipe_configs == 4) {
2446 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2447 switch (reg_offset) {
2449 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2450 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2451 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2452 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2455 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2456 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2457 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2458 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2461 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2462 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2463 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2464 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2467 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2468 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2469 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2470 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2473 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2474 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2475 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2476 TILE_SPLIT(split_equal_to_row_size));
2479 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2480 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2483 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2484 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2485 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2486 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2489 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2490 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2491 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2492 TILE_SPLIT(split_equal_to_row_size));
2495 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2496 PIPE_CONFIG(ADDR_SURF_P4_16x16));
2499 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2500 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2503 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2504 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2505 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2506 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2509 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2510 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2511 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2512 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2515 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2516 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2517 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2518 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2521 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2522 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2525 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2526 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2527 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2528 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2531 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2532 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2533 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2534 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2537 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2538 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2539 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2540 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2543 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2544 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2547 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2548 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2549 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2550 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2553 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2554 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2555 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2556 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2559 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2560 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2561 PIPE_CONFIG(ADDR_SURF_P4_16x16) |
2562 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2568 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2569 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2571 } else if (num_rbs < 4) {
2572 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2573 switch (reg_offset) {
2575 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2576 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2577 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2578 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2581 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2582 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2583 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2584 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2587 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2588 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2589 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2590 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2593 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2594 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2595 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2596 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2599 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2600 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2601 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2602 TILE_SPLIT(split_equal_to_row_size));
2605 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2606 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2609 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2610 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2611 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2612 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2615 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2616 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2617 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2618 TILE_SPLIT(split_equal_to_row_size));
2621 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2622 PIPE_CONFIG(ADDR_SURF_P4_8x16));
2625 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2626 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2629 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2630 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2631 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2632 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2635 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2636 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2637 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2638 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2641 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2642 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2643 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2644 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2647 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2648 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2651 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2652 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2653 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2654 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2657 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2658 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2659 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2660 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2663 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2664 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2665 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2666 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2669 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2670 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2673 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2674 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2675 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2676 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2679 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2680 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2681 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2682 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2685 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2686 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2687 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2688 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2694 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2695 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2698 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2699 switch (reg_offset) {
2701 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2702 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2703 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2704 NUM_BANKS(ADDR_SURF_16_BANK));
2707 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2708 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2709 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2710 NUM_BANKS(ADDR_SURF_16_BANK));
2713 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2714 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2715 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2716 NUM_BANKS(ADDR_SURF_16_BANK));
2719 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2720 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2721 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2722 NUM_BANKS(ADDR_SURF_16_BANK));
2725 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2726 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2727 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2728 NUM_BANKS(ADDR_SURF_16_BANK));
2731 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2732 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2733 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2734 NUM_BANKS(ADDR_SURF_8_BANK));
2737 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2738 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2739 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2740 NUM_BANKS(ADDR_SURF_4_BANK));
2743 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2744 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2745 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2746 NUM_BANKS(ADDR_SURF_16_BANK));
2749 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2750 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2751 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2752 NUM_BANKS(ADDR_SURF_16_BANK));
2755 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2756 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2757 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2758 NUM_BANKS(ADDR_SURF_16_BANK));
2761 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2762 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2763 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2764 NUM_BANKS(ADDR_SURF_16_BANK));
2767 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2768 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2769 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2770 NUM_BANKS(ADDR_SURF_16_BANK));
2773 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2774 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2775 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2776 NUM_BANKS(ADDR_SURF_8_BANK));
2779 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2780 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2781 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
2782 NUM_BANKS(ADDR_SURF_4_BANK));
2788 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
2789 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2791 } else if (num_pipe_configs == 2) {
2792 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2793 switch (reg_offset) {
2795 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2796 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2797 PIPE_CONFIG(ADDR_SURF_P2) |
2798 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
2801 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2802 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2803 PIPE_CONFIG(ADDR_SURF_P2) |
2804 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
2807 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2808 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2809 PIPE_CONFIG(ADDR_SURF_P2) |
2810 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2813 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2814 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2815 PIPE_CONFIG(ADDR_SURF_P2) |
2816 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
2819 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2820 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2821 PIPE_CONFIG(ADDR_SURF_P2) |
2822 TILE_SPLIT(split_equal_to_row_size));
2825 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2826 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
2829 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2830 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2831 PIPE_CONFIG(ADDR_SURF_P2) |
2832 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
2835 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2836 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
2837 PIPE_CONFIG(ADDR_SURF_P2) |
2838 TILE_SPLIT(split_equal_to_row_size));
2841 gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED);
2844 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2845 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
2848 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2849 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2850 PIPE_CONFIG(ADDR_SURF_P2) |
2851 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2854 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2855 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2856 PIPE_CONFIG(ADDR_SURF_P2) |
2857 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2860 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2861 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
2862 PIPE_CONFIG(ADDR_SURF_P2) |
2863 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2866 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2867 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
2870 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2871 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2872 PIPE_CONFIG(ADDR_SURF_P2) |
2873 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2876 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2877 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2878 PIPE_CONFIG(ADDR_SURF_P2) |
2879 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2882 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2883 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
2884 PIPE_CONFIG(ADDR_SURF_P2) |
2885 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2888 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2889 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
2892 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2893 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2894 PIPE_CONFIG(ADDR_SURF_P2) |
2895 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2898 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
2899 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2900 PIPE_CONFIG(ADDR_SURF_P2) |
2901 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2904 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
2905 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
2906 PIPE_CONFIG(ADDR_SURF_P2) |
2907 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
2913 rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
2914 WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2916 for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
2917 switch (reg_offset) {
2919 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2920 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2921 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2922 NUM_BANKS(ADDR_SURF_16_BANK));
2925 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2926 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2927 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2928 NUM_BANKS(ADDR_SURF_16_BANK));
2931 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2932 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2933 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2934 NUM_BANKS(ADDR_SURF_16_BANK));
2937 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2938 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2939 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2940 NUM_BANKS(ADDR_SURF_16_BANK));
2943 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2944 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2945 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2946 NUM_BANKS(ADDR_SURF_16_BANK));
2949 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2950 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2951 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2952 NUM_BANKS(ADDR_SURF_16_BANK));
2955 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2956 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2957 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
2958 NUM_BANKS(ADDR_SURF_8_BANK));
2961 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2962 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_8) |
2963 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2964 NUM_BANKS(ADDR_SURF_16_BANK));
2967 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_4) |
2968 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2969 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2970 NUM_BANKS(ADDR_SURF_16_BANK));
2973 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2974 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2975 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2976 NUM_BANKS(ADDR_SURF_16_BANK));
2979 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2980 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2981 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2982 NUM_BANKS(ADDR_SURF_16_BANK));
2985 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2986 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2987 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2988 NUM_BANKS(ADDR_SURF_16_BANK));
2991 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2992 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2993 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4) |
2994 NUM_BANKS(ADDR_SURF_16_BANK));
2997 gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2998 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2999 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
3000 NUM_BANKS(ADDR_SURF_8_BANK));
3006 rdev->config.cik.macrotile_mode_array[reg_offset] = gb_tile_moden;
3007 WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
3010 DRM_ERROR("unknown num pipe config: 0x%x\n", num_pipe_configs);
3014 * cik_select_se_sh - select which SE, SH to address
3016 * @rdev: radeon_device pointer
3017 * @se_num: shader engine to address
3018 * @sh_num: sh block to address
3020 * Select which SE, SH combinations to address. Certain
3021 * registers are instanced per SE or SH. 0xffffffff means
3022 * broadcast to all SEs or SHs (CIK).
3024 static void cik_select_se_sh(struct radeon_device *rdev,
3025 u32 se_num, u32 sh_num)
3027 u32 data = INSTANCE_BROADCAST_WRITES;
3029 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
3030 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
3031 else if (se_num == 0xffffffff)
3032 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
3033 else if (sh_num == 0xffffffff)
3034 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
3036 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
3037 WREG32(GRBM_GFX_INDEX, data);
3041 * cik_create_bitmask - create a bitmask
3043 * @bit_width: length of the mask
3045 * create a variable length bit mask (CIK).
3046 * Returns the bitmask.
3048 static u32 cik_create_bitmask(u32 bit_width)
3052 for (i = 0; i < bit_width; i++) {
3060 * cik_get_rb_disabled - computes the mask of disabled RBs
3062 * @rdev: radeon_device pointer
3063 * @max_rb_num: max RBs (render backends) for the asic
3064 * @se_num: number of SEs (shader engines) for the asic
3065 * @sh_per_se: number of SH blocks per SE for the asic
3067 * Calculates the bitmask of disabled RBs (CIK).
3068 * Returns the disabled RB bitmask.
3070 static u32 cik_get_rb_disabled(struct radeon_device *rdev,
3071 u32 max_rb_num_per_se,
3076 data = RREG32(CC_RB_BACKEND_DISABLE);
3078 data &= BACKEND_DISABLE_MASK;
3081 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3083 data >>= BACKEND_DISABLE_SHIFT;
3085 mask = cik_create_bitmask(max_rb_num_per_se / sh_per_se);
3091 * cik_setup_rb - setup the RBs on the asic
3093 * @rdev: radeon_device pointer
3094 * @se_num: number of SEs (shader engines) for the asic
3095 * @sh_per_se: number of SH blocks per SE for the asic
3096 * @max_rb_num: max RBs (render backends) for the asic
3098 * Configures per-SE/SH RB registers (CIK).
3100 static void cik_setup_rb(struct radeon_device *rdev,
3101 u32 se_num, u32 sh_per_se,
3102 u32 max_rb_num_per_se)
3106 u32 disabled_rbs = 0;
3107 u32 enabled_rbs = 0;
3109 for (i = 0; i < se_num; i++) {
3110 for (j = 0; j < sh_per_se; j++) {
3111 cik_select_se_sh(rdev, i, j);
3112 data = cik_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
3113 if (rdev->family == CHIP_HAWAII)
3114 disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
3116 disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
3119 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3122 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
3123 if (!(disabled_rbs & mask))
3124 enabled_rbs |= mask;
3128 rdev->config.cik.backend_enable_mask = enabled_rbs;
3130 for (i = 0; i < se_num; i++) {
3131 cik_select_se_sh(rdev, i, 0xffffffff);
3133 for (j = 0; j < sh_per_se; j++) {
3134 switch (enabled_rbs & 3) {
3137 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
3139 data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
3142 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3145 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3149 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3154 WREG32(PA_SC_RASTER_CONFIG, data);
3156 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3160 * cik_gpu_init - setup the 3D engine
3162 * @rdev: radeon_device pointer
3164 * Configures the 3D engine and tiling configuration
3165 * registers so that the 3D engine is usable.
3167 static void cik_gpu_init(struct radeon_device *rdev)
3169 u32 gb_addr_config = RREG32(GB_ADDR_CONFIG);
3170 u32 mc_shared_chmap, mc_arb_ramcfg;
3171 u32 hdp_host_path_cntl;
3175 switch (rdev->family) {
3177 rdev->config.cik.max_shader_engines = 2;
3178 rdev->config.cik.max_tile_pipes = 4;
3179 rdev->config.cik.max_cu_per_sh = 7;
3180 rdev->config.cik.max_sh_per_se = 1;
3181 rdev->config.cik.max_backends_per_se = 2;
3182 rdev->config.cik.max_texture_channel_caches = 4;
3183 rdev->config.cik.max_gprs = 256;
3184 rdev->config.cik.max_gs_threads = 32;
3185 rdev->config.cik.max_hw_contexts = 8;
3187 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3188 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3189 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3190 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3191 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3194 rdev->config.cik.max_shader_engines = 4;
3195 rdev->config.cik.max_tile_pipes = 16;
3196 rdev->config.cik.max_cu_per_sh = 11;
3197 rdev->config.cik.max_sh_per_se = 1;
3198 rdev->config.cik.max_backends_per_se = 4;
3199 rdev->config.cik.max_texture_channel_caches = 16;
3200 rdev->config.cik.max_gprs = 256;
3201 rdev->config.cik.max_gs_threads = 32;
3202 rdev->config.cik.max_hw_contexts = 8;
3204 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3205 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3206 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3207 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3208 gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
3211 rdev->config.cik.max_shader_engines = 1;
3212 rdev->config.cik.max_tile_pipes = 4;
3213 if ((rdev->pdev->device == 0x1304) ||
3214 (rdev->pdev->device == 0x1305) ||
3215 (rdev->pdev->device == 0x130C) ||
3216 (rdev->pdev->device == 0x130F) ||
3217 (rdev->pdev->device == 0x1310) ||
3218 (rdev->pdev->device == 0x1311) ||
3219 (rdev->pdev->device == 0x131C)) {
3220 rdev->config.cik.max_cu_per_sh = 8;
3221 rdev->config.cik.max_backends_per_se = 2;
3222 } else if ((rdev->pdev->device == 0x1309) ||
3223 (rdev->pdev->device == 0x130A) ||
3224 (rdev->pdev->device == 0x130D) ||
3225 (rdev->pdev->device == 0x1313) ||
3226 (rdev->pdev->device == 0x131D)) {
3227 rdev->config.cik.max_cu_per_sh = 6;
3228 rdev->config.cik.max_backends_per_se = 2;
3229 } else if ((rdev->pdev->device == 0x1306) ||
3230 (rdev->pdev->device == 0x1307) ||
3231 (rdev->pdev->device == 0x130B) ||
3232 (rdev->pdev->device == 0x130E) ||
3233 (rdev->pdev->device == 0x1315) ||
3234 (rdev->pdev->device == 0x1318) ||
3235 (rdev->pdev->device == 0x131B)) {
3236 rdev->config.cik.max_cu_per_sh = 4;
3237 rdev->config.cik.max_backends_per_se = 1;
3239 rdev->config.cik.max_cu_per_sh = 3;
3240 rdev->config.cik.max_backends_per_se = 1;
3242 rdev->config.cik.max_sh_per_se = 1;
3243 rdev->config.cik.max_texture_channel_caches = 4;
3244 rdev->config.cik.max_gprs = 256;
3245 rdev->config.cik.max_gs_threads = 16;
3246 rdev->config.cik.max_hw_contexts = 8;
3248 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3249 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3250 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3251 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3252 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3256 rdev->config.cik.max_shader_engines = 1;
3257 rdev->config.cik.max_tile_pipes = 2;
3258 rdev->config.cik.max_cu_per_sh = 2;
3259 rdev->config.cik.max_sh_per_se = 1;
3260 rdev->config.cik.max_backends_per_se = 1;
3261 rdev->config.cik.max_texture_channel_caches = 2;
3262 rdev->config.cik.max_gprs = 256;
3263 rdev->config.cik.max_gs_threads = 16;
3264 rdev->config.cik.max_hw_contexts = 8;
3266 rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
3267 rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
3268 rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
3269 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
3270 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
3274 /* Initialize HDP */
3275 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3276 WREG32((0x2c14 + j), 0x00000000);
3277 WREG32((0x2c18 + j), 0x00000000);
3278 WREG32((0x2c1c + j), 0x00000000);
3279 WREG32((0x2c20 + j), 0x00000000);
3280 WREG32((0x2c24 + j), 0x00000000);
3283 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3285 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3287 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3288 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3290 rdev->config.cik.num_tile_pipes = rdev->config.cik.max_tile_pipes;
3291 rdev->config.cik.mem_max_burst_length_bytes = 256;
3292 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3293 rdev->config.cik.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3294 if (rdev->config.cik.mem_row_size_in_kb > 4)
3295 rdev->config.cik.mem_row_size_in_kb = 4;
3296 /* XXX use MC settings? */
3297 rdev->config.cik.shader_engine_tile_size = 32;
3298 rdev->config.cik.num_gpus = 1;
3299 rdev->config.cik.multi_gpu_tile_size = 64;
3301 /* fix up row size */
3302 gb_addr_config &= ~ROW_SIZE_MASK;
3303 switch (rdev->config.cik.mem_row_size_in_kb) {
3306 gb_addr_config |= ROW_SIZE(0);
3309 gb_addr_config |= ROW_SIZE(1);
3312 gb_addr_config |= ROW_SIZE(2);
3316 /* setup tiling info dword. gb_addr_config is not adequate since it does
3317 * not have bank info, so create a custom tiling dword.
3318 * bits 3:0 num_pipes
3319 * bits 7:4 num_banks
3320 * bits 11:8 group_size
3321 * bits 15:12 row_size
3323 rdev->config.cik.tile_config = 0;
3324 switch (rdev->config.cik.num_tile_pipes) {
3326 rdev->config.cik.tile_config |= (0 << 0);
3329 rdev->config.cik.tile_config |= (1 << 0);
3332 rdev->config.cik.tile_config |= (2 << 0);
3336 /* XXX what about 12? */
3337 rdev->config.cik.tile_config |= (3 << 0);
3340 rdev->config.cik.tile_config |=
3341 ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) << 4;
3342 rdev->config.cik.tile_config |=
3343 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3344 rdev->config.cik.tile_config |=
3345 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3347 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3348 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3349 WREG32(DMIF_ADDR_CALC, gb_addr_config);
3350 WREG32(SDMA0_TILING_CONFIG + SDMA0_REGISTER_OFFSET, gb_addr_config & 0x70);
3351 WREG32(SDMA0_TILING_CONFIG + SDMA1_REGISTER_OFFSET, gb_addr_config & 0x70);
3352 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3353 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3354 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3356 cik_tiling_mode_table_init(rdev);
3358 cik_setup_rb(rdev, rdev->config.cik.max_shader_engines,
3359 rdev->config.cik.max_sh_per_se,
3360 rdev->config.cik.max_backends_per_se);
3362 /* set HW defaults for 3D engine */
3363 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3365 WREG32(SX_DEBUG_1, 0x20);
3367 WREG32(TA_CNTL_AUX, 0x00010000);
3369 tmp = RREG32(SPI_CONFIG_CNTL);
3371 WREG32(SPI_CONFIG_CNTL, tmp);
3373 WREG32(SQ_CONFIG, 1);
3375 WREG32(DB_DEBUG, 0);
3377 tmp = RREG32(DB_DEBUG2) & ~0xf00fffff;
3379 WREG32(DB_DEBUG2, tmp);
3381 tmp = RREG32(DB_DEBUG3) & ~0x0002021c;
3383 WREG32(DB_DEBUG3, tmp);
3385 tmp = RREG32(CB_HW_CONTROL) & ~0x00010000;
3387 WREG32(CB_HW_CONTROL, tmp);
3389 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3391 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_frontend) |
3392 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.cik.sc_prim_fifo_size_backend) |
3393 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cik.sc_hiz_tile_fifo_size) |
3394 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cik.sc_earlyz_tile_fifo_size)));
3396 WREG32(VGT_NUM_INSTANCES, 1);
3398 WREG32(CP_PERFMON_CNTL, 0);
3400 WREG32(SQ_CONFIG, 0);
3402 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3403 FORCE_EOV_MAX_REZ_CNT(255)));
3405 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3406 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3408 WREG32(VGT_GS_VERTEX_REUSE, 16);
3409 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3411 tmp = RREG32(HDP_MISC_CNTL);
3412 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3413 WREG32(HDP_MISC_CNTL, tmp);
3415 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3416 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3418 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3419 WREG32(PA_SC_ENHANCE, ENABLE_PA_SC_OUT_OF_ORDER);
3425 * GPU scratch registers helpers function.
3428 * cik_scratch_init - setup driver info for CP scratch regs
3430 * @rdev: radeon_device pointer
3432 * Set up the number and offset of the CP scratch registers.
3433 * NOTE: use of CP scratch registers is a legacy inferface and
3434 * is not used by default on newer asics (r6xx+). On newer asics,
3435 * memory buffers are used for fences rather than scratch regs.
3437 static void cik_scratch_init(struct radeon_device *rdev)
3441 rdev->scratch.num_reg = 7;
3442 rdev->scratch.reg_base = SCRATCH_REG0;
3443 for (i = 0; i < rdev->scratch.num_reg; i++) {
3444 rdev->scratch.free[i] = true;
3445 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3450 * cik_ring_test - basic gfx ring test
3452 * @rdev: radeon_device pointer
3453 * @ring: radeon_ring structure holding ring information
3455 * Allocate a scratch register and write to it using the gfx ring (CIK).
3456 * Provides a basic gfx ring test to verify that the ring is working.
3457 * Used by cik_cp_gfx_resume();
3458 * Returns 0 on success, error on failure.
3460 int cik_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
3467 r = radeon_scratch_get(rdev, &scratch);
3469 DRM_ERROR("radeon: cp failed to get scratch reg (%d).\n", r);
3472 WREG32(scratch, 0xCAFEDEAD);
3473 r = radeon_ring_lock(rdev, ring, 3);
3475 DRM_ERROR("radeon: cp failed to lock ring %d (%d).\n", ring->idx, r);
3476 radeon_scratch_free(rdev, scratch);
3479 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3480 radeon_ring_write(ring, ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2));
3481 radeon_ring_write(ring, 0xDEADBEEF);
3482 radeon_ring_unlock_commit(rdev, ring);
3484 for (i = 0; i < rdev->usec_timeout; i++) {
3485 tmp = RREG32(scratch);
3486 if (tmp == 0xDEADBEEF)
3490 if (i < rdev->usec_timeout) {
3491 DRM_INFO("ring test on %d succeeded in %d usecs\n", ring->idx, i);
3493 DRM_ERROR("radeon: ring %d test failed (scratch(0x%04X)=0x%08X)\n",
3494 ring->idx, scratch, tmp);
3497 radeon_scratch_free(rdev, scratch);
3502 * cik_hdp_flush_cp_ring_emit - emit an hdp flush on the cp
3504 * @rdev: radeon_device pointer
3505 * @ridx: radeon ring index
3507 * Emits an hdp flush on the cp.
3509 static void cik_hdp_flush_cp_ring_emit(struct radeon_device *rdev,
3512 struct radeon_ring *ring = &rdev->ring[ridx];
3515 switch (ring->idx) {
3516 case CAYMAN_RING_TYPE_CP1_INDEX:
3517 case CAYMAN_RING_TYPE_CP2_INDEX:
3521 ref_and_mask = CP2 << ring->pipe;
3524 ref_and_mask = CP6 << ring->pipe;
3530 case RADEON_RING_TYPE_GFX_INDEX:
3535 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
3536 radeon_ring_write(ring, (WAIT_REG_MEM_OPERATION(1) | /* write, wait, write */
3537 WAIT_REG_MEM_FUNCTION(3) | /* == */
3538 WAIT_REG_MEM_ENGINE(1))); /* pfp */
3539 radeon_ring_write(ring, GPU_HDP_FLUSH_REQ >> 2);
3540 radeon_ring_write(ring, GPU_HDP_FLUSH_DONE >> 2);
3541 radeon_ring_write(ring, ref_and_mask);
3542 radeon_ring_write(ring, ref_and_mask);
3543 radeon_ring_write(ring, 0x20); /* poll interval */
3547 * cik_fence_gfx_ring_emit - emit a fence on the gfx ring
3549 * @rdev: radeon_device pointer
3550 * @fence: radeon fence object
3552 * Emits a fence sequnce number on the gfx ring and flushes
3555 void cik_fence_gfx_ring_emit(struct radeon_device *rdev,
3556 struct radeon_fence *fence)
3558 struct radeon_ring *ring = &rdev->ring[fence->ring];
3559 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3561 /* EVENT_WRITE_EOP - flush caches, send int */
3562 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3563 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3565 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3567 radeon_ring_write(ring, addr & 0xfffffffc);
3568 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | DATA_SEL(1) | INT_SEL(2));
3569 radeon_ring_write(ring, fence->seq);
3570 radeon_ring_write(ring, 0);
3572 cik_hdp_flush_cp_ring_emit(rdev, fence->ring);
3576 * cik_fence_compute_ring_emit - emit a fence on the compute ring
3578 * @rdev: radeon_device pointer
3579 * @fence: radeon fence object
3581 * Emits a fence sequnce number on the compute ring and flushes
3584 void cik_fence_compute_ring_emit(struct radeon_device *rdev,
3585 struct radeon_fence *fence)
3587 struct radeon_ring *ring = &rdev->ring[fence->ring];
3588 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3590 /* RELEASE_MEM - flush caches, send int */
3591 radeon_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 5));
3592 radeon_ring_write(ring, (EOP_TCL1_ACTION_EN |
3594 EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
3596 radeon_ring_write(ring, DATA_SEL(1) | INT_SEL(2));
3597 radeon_ring_write(ring, addr & 0xfffffffc);
3598 radeon_ring_write(ring, upper_32_bits(addr));
3599 radeon_ring_write(ring, fence->seq);
3600 radeon_ring_write(ring, 0);
3602 cik_hdp_flush_cp_ring_emit(rdev, fence->ring);
3605 bool cik_semaphore_ring_emit(struct radeon_device *rdev,
3606 struct radeon_ring *ring,
3607 struct radeon_semaphore *semaphore,
3610 uint64_t addr = semaphore->gpu_addr;
3611 unsigned sel = emit_wait ? PACKET3_SEM_SEL_WAIT : PACKET3_SEM_SEL_SIGNAL;
3613 radeon_ring_write(ring, PACKET3(PACKET3_MEM_SEMAPHORE, 1));
3614 radeon_ring_write(ring, addr & 0xffffffff);
3615 radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
3621 * cik_copy_cpdma - copy pages using the CP DMA engine
3623 * @rdev: radeon_device pointer
3624 * @src_offset: src GPU address
3625 * @dst_offset: dst GPU address
3626 * @num_gpu_pages: number of GPU pages to xfer
3627 * @fence: radeon fence object
3629 * Copy GPU paging using the CP DMA engine (CIK+).
3630 * Used by the radeon ttm implementation to move pages if
3631 * registered as the asic copy callback.
3633 int cik_copy_cpdma(struct radeon_device *rdev,
3634 uint64_t src_offset, uint64_t dst_offset,
3635 unsigned num_gpu_pages,
3636 struct radeon_fence **fence)
3638 struct radeon_semaphore *sem = NULL;
3639 int ring_index = rdev->asic->copy.blit_ring_index;
3640 struct radeon_ring *ring = &rdev->ring[ring_index];
3641 u32 size_in_bytes, cur_size_in_bytes, control;
3645 r = radeon_semaphore_create(rdev, &sem);
3647 DRM_ERROR("radeon: moving bo (%d).\n", r);
3651 size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
3652 num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
3653 r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
3655 DRM_ERROR("radeon: moving bo (%d).\n", r);
3656 radeon_semaphore_free(rdev, &sem, NULL);
3660 radeon_semaphore_sync_to(sem, *fence);
3661 radeon_semaphore_sync_rings(rdev, sem, ring->idx);
3663 for (i = 0; i < num_loops; i++) {
3664 cur_size_in_bytes = size_in_bytes;
3665 if (cur_size_in_bytes > 0x1fffff)
3666 cur_size_in_bytes = 0x1fffff;
3667 size_in_bytes -= cur_size_in_bytes;
3669 if (size_in_bytes == 0)
3670 control |= PACKET3_DMA_DATA_CP_SYNC;
3671 radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
3672 radeon_ring_write(ring, control);
3673 radeon_ring_write(ring, lower_32_bits(src_offset));
3674 radeon_ring_write(ring, upper_32_bits(src_offset));
3675 radeon_ring_write(ring, lower_32_bits(dst_offset));
3676 radeon_ring_write(ring, upper_32_bits(dst_offset));
3677 radeon_ring_write(ring, cur_size_in_bytes);
3678 src_offset += cur_size_in_bytes;
3679 dst_offset += cur_size_in_bytes;
3682 r = radeon_fence_emit(rdev, fence, ring->idx);
3684 radeon_ring_unlock_undo(rdev, ring);
3688 radeon_ring_unlock_commit(rdev, ring);
3689 radeon_semaphore_free(rdev, &sem, *fence);
3698 * cik_ring_ib_execute - emit an IB (Indirect Buffer) on the gfx ring
3700 * @rdev: radeon_device pointer
3701 * @ib: radeon indirect buffer object
3703 * Emits an DE (drawing engine) or CE (constant engine) IB
3704 * on the gfx ring. IBs are usually generated by userspace
3705 * acceleration drivers and submitted to the kernel for
3706 * sheduling on the ring. This function schedules the IB
3707 * on the gfx ring for execution by the GPU.
3709 void cik_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3711 struct radeon_ring *ring = &rdev->ring[ib->ring];
3712 u32 header, control = INDIRECT_BUFFER_VALID;
3714 if (ib->is_const_ib) {
3715 /* set switch buffer packet before const IB */
3716 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3717 radeon_ring_write(ring, 0);
3719 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3722 if (ring->rptr_save_reg) {
3723 next_rptr = ring->wptr + 3 + 4;
3724 radeon_ring_write(ring, PACKET3(PACKET3_SET_UCONFIG_REG, 1));
3725 radeon_ring_write(ring, ((ring->rptr_save_reg -
3726 PACKET3_SET_UCONFIG_REG_START) >> 2));
3727 radeon_ring_write(ring, next_rptr);
3728 } else if (rdev->wb.enabled) {
3729 next_rptr = ring->wptr + 5 + 4;
3730 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3731 radeon_ring_write(ring, WRITE_DATA_DST_SEL(1));
3732 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3733 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
3734 radeon_ring_write(ring, next_rptr);
3737 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3740 control |= ib->length_dw |
3741 (ib->vm ? (ib->vm->id << 24) : 0);
3743 radeon_ring_write(ring, header);
3744 radeon_ring_write(ring,
3748 (ib->gpu_addr & 0xFFFFFFFC));
3749 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3750 radeon_ring_write(ring, control);
3754 * cik_ib_test - basic gfx ring IB test
3756 * @rdev: radeon_device pointer
3757 * @ring: radeon_ring structure holding ring information
3759 * Allocate an IB and execute it on the gfx ring (CIK).
3760 * Provides a basic gfx ring test to verify that IBs are working.
3761 * Returns 0 on success, error on failure.
3763 int cik_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
3765 struct radeon_ib ib;
3771 r = radeon_scratch_get(rdev, &scratch);
3773 DRM_ERROR("radeon: failed to get scratch reg (%d).\n", r);
3776 WREG32(scratch, 0xCAFEDEAD);
3777 r = radeon_ib_get(rdev, ring->idx, &ib, NULL, 256);
3779 DRM_ERROR("radeon: failed to get ib (%d).\n", r);
3780 radeon_scratch_free(rdev, scratch);
3783 ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
3784 ib.ptr[1] = ((scratch - PACKET3_SET_UCONFIG_REG_START) >> 2);
3785 ib.ptr[2] = 0xDEADBEEF;
3787 r = radeon_ib_schedule(rdev, &ib, NULL);
3789 radeon_scratch_free(rdev, scratch);
3790 radeon_ib_free(rdev, &ib);
3791 DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
3794 r = radeon_fence_wait(ib.fence, false);
3796 DRM_ERROR("radeon: fence wait failed (%d).\n", r);
3797 radeon_scratch_free(rdev, scratch);
3798 radeon_ib_free(rdev, &ib);
3801 for (i = 0; i < rdev->usec_timeout; i++) {
3802 tmp = RREG32(scratch);
3803 if (tmp == 0xDEADBEEF)
3807 if (i < rdev->usec_timeout) {
3808 DRM_INFO("ib test on ring %d succeeded in %u usecs\n", ib.fence->ring, i);
3810 DRM_ERROR("radeon: ib test failed (scratch(0x%04X)=0x%08X)\n",
3814 radeon_scratch_free(rdev, scratch);
3815 radeon_ib_free(rdev, &ib);
3821 * On CIK, gfx and compute now have independant command processors.
3824 * Gfx consists of a single ring and can process both gfx jobs and
3825 * compute jobs. The gfx CP consists of three microengines (ME):
3826 * PFP - Pre-Fetch Parser
3828 * CE - Constant Engine
3829 * The PFP and ME make up what is considered the Drawing Engine (DE).
3830 * The CE is an asynchronous engine used for updating buffer desciptors
3831 * used by the DE so that they can be loaded into cache in parallel
3832 * while the DE is processing state update packets.
3835 * The compute CP consists of two microengines (ME):
3836 * MEC1 - Compute MicroEngine 1
3837 * MEC2 - Compute MicroEngine 2
3838 * Each MEC supports 4 compute pipes and each pipe supports 8 queues.
3839 * The queues are exposed to userspace and are programmed directly
3840 * by the compute runtime.
3843 * cik_cp_gfx_enable - enable/disable the gfx CP MEs
3845 * @rdev: radeon_device pointer
3846 * @enable: enable or disable the MEs
3848 * Halts or unhalts the gfx MEs.
3850 static void cik_cp_gfx_enable(struct radeon_device *rdev, bool enable)
3853 WREG32(CP_ME_CNTL, 0);
3855 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3856 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3857 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3858 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3864 * cik_cp_gfx_load_microcode - load the gfx CP ME ucode
3866 * @rdev: radeon_device pointer
3868 * Loads the gfx PFP, ME, and CE ucode.
3869 * Returns 0 for success, -EINVAL if the ucode is not available.
3871 static int cik_cp_gfx_load_microcode(struct radeon_device *rdev)
3873 const __be32 *fw_data;
3876 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3879 cik_cp_gfx_enable(rdev, false);
3882 fw_data = (const __be32 *)rdev->pfp_fw->data;
3883 WREG32(CP_PFP_UCODE_ADDR, 0);
3884 for (i = 0; i < CIK_PFP_UCODE_SIZE; i++)
3885 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3886 WREG32(CP_PFP_UCODE_ADDR, 0);
3889 fw_data = (const __be32 *)rdev->ce_fw->data;
3890 WREG32(CP_CE_UCODE_ADDR, 0);
3891 for (i = 0; i < CIK_CE_UCODE_SIZE; i++)
3892 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3893 WREG32(CP_CE_UCODE_ADDR, 0);
3896 fw_data = (const __be32 *)rdev->me_fw->data;
3897 WREG32(CP_ME_RAM_WADDR, 0);
3898 for (i = 0; i < CIK_ME_UCODE_SIZE; i++)
3899 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3900 WREG32(CP_ME_RAM_WADDR, 0);
3902 WREG32(CP_PFP_UCODE_ADDR, 0);
3903 WREG32(CP_CE_UCODE_ADDR, 0);
3904 WREG32(CP_ME_RAM_WADDR, 0);
3905 WREG32(CP_ME_RAM_RADDR, 0);
3910 * cik_cp_gfx_start - start the gfx ring
3912 * @rdev: radeon_device pointer
3914 * Enables the ring and loads the clear state context and other
3915 * packets required to init the ring.
3916 * Returns 0 for success, error for failure.
3918 static int cik_cp_gfx_start(struct radeon_device *rdev)
3920 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3924 WREG32(CP_MAX_CONTEXT, rdev->config.cik.max_hw_contexts - 1);
3925 WREG32(CP_ENDIAN_SWAP, 0);
3926 WREG32(CP_DEVICE_ID, 1);
3928 cik_cp_gfx_enable(rdev, true);
3930 r = radeon_ring_lock(rdev, ring, cik_default_size + 17);
3932 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3936 /* init the CE partitions. CE only used for gfx on CIK */
3937 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3938 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3939 radeon_ring_write(ring, 0xc000);
3940 radeon_ring_write(ring, 0xc000);
3942 /* setup clear context state */
3943 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3944 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3946 radeon_ring_write(ring, PACKET3(PACKET3_CONTEXT_CONTROL, 1));
3947 radeon_ring_write(ring, 0x80000000);
3948 radeon_ring_write(ring, 0x80000000);
3950 for (i = 0; i < cik_default_size; i++)
3951 radeon_ring_write(ring, cik_default_state[i]);
3953 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3954 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3956 /* set clear context state */
3957 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3958 radeon_ring_write(ring, 0);
3960 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3961 radeon_ring_write(ring, 0x00000316);
3962 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3963 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3965 radeon_ring_unlock_commit(rdev, ring);
3971 * cik_cp_gfx_fini - stop the gfx ring
3973 * @rdev: radeon_device pointer
3975 * Stop the gfx ring and tear down the driver ring
3978 static void cik_cp_gfx_fini(struct radeon_device *rdev)
3980 cik_cp_gfx_enable(rdev, false);
3981 radeon_ring_fini(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3985 * cik_cp_gfx_resume - setup the gfx ring buffer registers
3987 * @rdev: radeon_device pointer
3989 * Program the location and size of the gfx ring buffer
3990 * and test it to make sure it's working.
3991 * Returns 0 for success, error for failure.
3993 static int cik_cp_gfx_resume(struct radeon_device *rdev)
3995 struct radeon_ring *ring;
4001 WREG32(CP_SEM_WAIT_TIMER, 0x0);
4002 if (rdev->family != CHIP_HAWAII)
4003 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
4005 /* Set the write pointer delay */
4006 WREG32(CP_RB_WPTR_DELAY, 0);
4008 /* set the RB to use vmid 0 */
4009 WREG32(CP_RB_VMID, 0);
4011 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
4013 /* ring 0 - compute and gfx */
4014 /* Set ring buffer size */
4015 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
4016 rb_bufsz = order_base_2(ring->ring_size / 8);
4017 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
4019 tmp |= BUF_SWAP_32BIT;
4021 WREG32(CP_RB0_CNTL, tmp);
4023 /* Initialize the ring buffer's read and write pointers */
4024 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
4026 WREG32(CP_RB0_WPTR, ring->wptr);
4028 /* set the wb address wether it's enabled or not */
4029 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
4030 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
4032 /* scratch register shadowing is no longer supported */
4033 WREG32(SCRATCH_UMSK, 0);
4035 if (!rdev->wb.enabled)
4036 tmp |= RB_NO_UPDATE;
4039 WREG32(CP_RB0_CNTL, tmp);
4041 rb_addr = ring->gpu_addr >> 8;
4042 WREG32(CP_RB0_BASE, rb_addr);
4043 WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
4045 ring->rptr = RREG32(CP_RB0_RPTR);
4047 /* start the ring */
4048 cik_cp_gfx_start(rdev);
4049 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
4050 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
4052 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
4056 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
4057 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
4062 u32 cik_gfx_get_rptr(struct radeon_device *rdev,
4063 struct radeon_ring *ring)
4067 if (rdev->wb.enabled)
4068 rptr = rdev->wb.wb[ring->rptr_offs/4];
4070 rptr = RREG32(CP_RB0_RPTR);
4075 u32 cik_gfx_get_wptr(struct radeon_device *rdev,
4076 struct radeon_ring *ring)
4080 wptr = RREG32(CP_RB0_WPTR);
4085 void cik_gfx_set_wptr(struct radeon_device *rdev,
4086 struct radeon_ring *ring)
4088 WREG32(CP_RB0_WPTR, ring->wptr);
4089 (void)RREG32(CP_RB0_WPTR);
4092 u32 cik_compute_get_rptr(struct radeon_device *rdev,
4093 struct radeon_ring *ring)
4097 if (rdev->wb.enabled) {
4098 rptr = rdev->wb.wb[ring->rptr_offs/4];
4100 mutex_lock(&rdev->srbm_mutex);
4101 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4102 rptr = RREG32(CP_HQD_PQ_RPTR);
4103 cik_srbm_select(rdev, 0, 0, 0, 0);
4104 mutex_unlock(&rdev->srbm_mutex);
4110 u32 cik_compute_get_wptr(struct radeon_device *rdev,
4111 struct radeon_ring *ring)
4115 if (rdev->wb.enabled) {
4116 /* XXX check if swapping is necessary on BE */
4117 wptr = rdev->wb.wb[ring->wptr_offs/4];
4119 mutex_lock(&rdev->srbm_mutex);
4120 cik_srbm_select(rdev, ring->me, ring->pipe, ring->queue, 0);
4121 wptr = RREG32(CP_HQD_PQ_WPTR);
4122 cik_srbm_select(rdev, 0, 0, 0, 0);
4123 mutex_unlock(&rdev->srbm_mutex);
4129 void cik_compute_set_wptr(struct radeon_device *rdev,
4130 struct radeon_ring *ring)
4132 /* XXX check if swapping is necessary on BE */
4133 rdev->wb.wb[ring->wptr_offs/4] = ring->wptr;
4134 WDOORBELL32(ring->doorbell_index, ring->wptr);
4138 * cik_cp_compute_enable - enable/disable the compute CP MEs
4140 * @rdev: radeon_device pointer
4141 * @enable: enable or disable the MEs
4143 * Halts or unhalts the compute MEs.
4145 static void cik_cp_compute_enable(struct radeon_device *rdev, bool enable)
4148 WREG32(CP_MEC_CNTL, 0);
4150 WREG32(CP_MEC_CNTL, (MEC_ME1_HALT | MEC_ME2_HALT));
4151 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
4152 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
4158 * cik_cp_compute_load_microcode - load the compute CP ME ucode
4160 * @rdev: radeon_device pointer
4162 * Loads the compute MEC1&2 ucode.
4163 * Returns 0 for success, -EINVAL if the ucode is not available.
4165 static int cik_cp_compute_load_microcode(struct radeon_device *rdev)
4167 const __be32 *fw_data;
4173 cik_cp_compute_enable(rdev, false);
4176 fw_data = (const __be32 *)rdev->mec_fw->data;
4177 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4178 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4179 WREG32(CP_MEC_ME1_UCODE_DATA, be32_to_cpup(fw_data++));
4180 WREG32(CP_MEC_ME1_UCODE_ADDR, 0);
4182 if (rdev->family == CHIP_KAVERI) {
4184 fw_data = (const __be32 *)rdev->mec_fw->data;
4185 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4186 for (i = 0; i < CIK_MEC_UCODE_SIZE; i++)
4187 WREG32(CP_MEC_ME2_UCODE_DATA, be32_to_cpup(fw_data++));
4188 WREG32(CP_MEC_ME2_UCODE_ADDR, 0);
4195 * cik_cp_compute_start - start the compute queues
4197 * @rdev: radeon_device pointer
4199 * Enable the compute queues.
4200 * Returns 0 for success, error for failure.
4202 static int cik_cp_compute_start(struct radeon_device *rdev)
4204 cik_cp_compute_enable(rdev, true);
4210 * cik_cp_compute_fini - stop the compute queues
4212 * @rdev: radeon_device pointer
4214 * Stop the compute queues and tear down the driver queue
4217 static void cik_cp_compute_fini(struct radeon_device *rdev)
4221 cik_cp_compute_enable(rdev, false);
4223 for (i = 0; i < 2; i++) {
4225 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4227 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4229 if (rdev->ring[idx].mqd_obj) {
4230 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4231 if (unlikely(r != 0))
4232 dev_warn(rdev->dev, "(%d) reserve MQD bo failed\n", r);
4234 radeon_bo_unpin(rdev->ring[idx].mqd_obj);
4235 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4237 radeon_bo_unref(&rdev->ring[idx].mqd_obj);
4238 rdev->ring[idx].mqd_obj = NULL;
4243 static void cik_mec_fini(struct radeon_device *rdev)
4247 if (rdev->mec.hpd_eop_obj) {
4248 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4249 if (unlikely(r != 0))
4250 dev_warn(rdev->dev, "(%d) reserve HPD EOP bo failed\n", r);
4251 radeon_bo_unpin(rdev->mec.hpd_eop_obj);
4252 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4254 radeon_bo_unref(&rdev->mec.hpd_eop_obj);
4255 rdev->mec.hpd_eop_obj = NULL;
4259 #define MEC_HPD_SIZE 2048
4261 static int cik_mec_init(struct radeon_device *rdev)
4267 * KV: 2 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 64 Queues total
4268 * CI/KB: 1 MEC, 4 Pipes/MEC, 8 Queues/Pipe - 32 Queues total
4270 if (rdev->family == CHIP_KAVERI)
4271 rdev->mec.num_mec = 2;
4273 rdev->mec.num_mec = 1;
4274 rdev->mec.num_pipe = 4;
4275 rdev->mec.num_queue = rdev->mec.num_mec * rdev->mec.num_pipe * 8;
4277 if (rdev->mec.hpd_eop_obj == NULL) {
4278 r = radeon_bo_create(rdev,
4279 rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2,
4281 RADEON_GEM_DOMAIN_GTT, NULL,
4282 &rdev->mec.hpd_eop_obj);
4284 dev_warn(rdev->dev, "(%d) create HDP EOP bo failed\n", r);
4289 r = radeon_bo_reserve(rdev->mec.hpd_eop_obj, false);
4290 if (unlikely(r != 0)) {
4294 r = radeon_bo_pin(rdev->mec.hpd_eop_obj, RADEON_GEM_DOMAIN_GTT,
4295 &rdev->mec.hpd_eop_gpu_addr);
4297 dev_warn(rdev->dev, "(%d) pin HDP EOP bo failed\n", r);
4301 r = radeon_bo_kmap(rdev->mec.hpd_eop_obj, (void **)&hpd);
4303 dev_warn(rdev->dev, "(%d) map HDP EOP bo failed\n", r);
4308 /* clear memory. Not sure if this is required or not */
4309 memset(hpd, 0, rdev->mec.num_mec *rdev->mec.num_pipe * MEC_HPD_SIZE * 2);
4311 radeon_bo_kunmap(rdev->mec.hpd_eop_obj);
4312 radeon_bo_unreserve(rdev->mec.hpd_eop_obj);
4317 struct hqd_registers
4319 u32 cp_mqd_base_addr;
4320 u32 cp_mqd_base_addr_hi;
4323 u32 cp_hqd_persistent_state;
4324 u32 cp_hqd_pipe_priority;
4325 u32 cp_hqd_queue_priority;
4328 u32 cp_hqd_pq_base_hi;
4330 u32 cp_hqd_pq_rptr_report_addr;
4331 u32 cp_hqd_pq_rptr_report_addr_hi;
4332 u32 cp_hqd_pq_wptr_poll_addr;
4333 u32 cp_hqd_pq_wptr_poll_addr_hi;
4334 u32 cp_hqd_pq_doorbell_control;
4336 u32 cp_hqd_pq_control;
4337 u32 cp_hqd_ib_base_addr;
4338 u32 cp_hqd_ib_base_addr_hi;
4340 u32 cp_hqd_ib_control;
4341 u32 cp_hqd_iq_timer;
4343 u32 cp_hqd_dequeue_request;
4344 u32 cp_hqd_dma_offload;
4345 u32 cp_hqd_sema_cmd;
4346 u32 cp_hqd_msg_type;
4347 u32 cp_hqd_atomic0_preop_lo;
4348 u32 cp_hqd_atomic0_preop_hi;
4349 u32 cp_hqd_atomic1_preop_lo;
4350 u32 cp_hqd_atomic1_preop_hi;
4351 u32 cp_hqd_hq_scheduler0;
4352 u32 cp_hqd_hq_scheduler1;
4359 u32 dispatch_initiator;
4363 u32 pipeline_stat_enable;
4364 u32 perf_counter_enable;
4370 u32 resource_limits;
4371 u32 static_thread_mgmt01[2];
4373 u32 static_thread_mgmt23[2];
4375 u32 thread_trace_enable;
4378 u32 vgtcs_invoke_count[2];
4379 struct hqd_registers queue_state;
4381 u32 interrupt_queue[64];
4385 * cik_cp_compute_resume - setup the compute queue registers
4387 * @rdev: radeon_device pointer
4389 * Program the compute queues and test them to make sure they
4391 * Returns 0 for success, error for failure.
4393 static int cik_cp_compute_resume(struct radeon_device *rdev)
4397 bool use_doorbell = true;
4403 struct bonaire_mqd *mqd;
4405 r = cik_cp_compute_start(rdev);
4409 /* fix up chicken bits */
4410 tmp = RREG32(CP_CPF_DEBUG);
4412 WREG32(CP_CPF_DEBUG, tmp);
4414 /* init the pipes */
4415 mutex_lock(&rdev->srbm_mutex);
4416 for (i = 0; i < (rdev->mec.num_pipe * rdev->mec.num_mec); i++) {
4417 int me = (i < 4) ? 1 : 2;
4418 int pipe = (i < 4) ? i : (i - 4);
4420 eop_gpu_addr = rdev->mec.hpd_eop_gpu_addr + (i * MEC_HPD_SIZE * 2);
4422 cik_srbm_select(rdev, me, pipe, 0, 0);
4424 /* write the EOP addr */
4425 WREG32(CP_HPD_EOP_BASE_ADDR, eop_gpu_addr >> 8);
4426 WREG32(CP_HPD_EOP_BASE_ADDR_HI, upper_32_bits(eop_gpu_addr) >> 8);
4428 /* set the VMID assigned */
4429 WREG32(CP_HPD_EOP_VMID, 0);
4431 /* set the EOP size, register value is 2^(EOP_SIZE+1) dwords */
4432 tmp = RREG32(CP_HPD_EOP_CONTROL);
4433 tmp &= ~EOP_SIZE_MASK;
4434 tmp |= order_base_2(MEC_HPD_SIZE / 8);
4435 WREG32(CP_HPD_EOP_CONTROL, tmp);
4437 cik_srbm_select(rdev, 0, 0, 0, 0);
4438 mutex_unlock(&rdev->srbm_mutex);
4440 /* init the queues. Just two for now. */
4441 for (i = 0; i < 2; i++) {
4443 idx = CAYMAN_RING_TYPE_CP1_INDEX;
4445 idx = CAYMAN_RING_TYPE_CP2_INDEX;
4447 if (rdev->ring[idx].mqd_obj == NULL) {
4448 r = radeon_bo_create(rdev,
4449 sizeof(struct bonaire_mqd),
4451 RADEON_GEM_DOMAIN_GTT, NULL,
4452 &rdev->ring[idx].mqd_obj);
4454 dev_warn(rdev->dev, "(%d) create MQD bo failed\n", r);
4459 r = radeon_bo_reserve(rdev->ring[idx].mqd_obj, false);
4460 if (unlikely(r != 0)) {
4461 cik_cp_compute_fini(rdev);
4464 r = radeon_bo_pin(rdev->ring[idx].mqd_obj, RADEON_GEM_DOMAIN_GTT,
4467 dev_warn(rdev->dev, "(%d) pin MQD bo failed\n", r);
4468 cik_cp_compute_fini(rdev);
4471 r = radeon_bo_kmap(rdev->ring[idx].mqd_obj, (void **)&buf);
4473 dev_warn(rdev->dev, "(%d) map MQD bo failed\n", r);
4474 cik_cp_compute_fini(rdev);
4478 /* init the mqd struct */
4479 memset(buf, 0, sizeof(struct bonaire_mqd));
4481 mqd = (struct bonaire_mqd *)buf;
4482 mqd->header = 0xC0310800;
4483 mqd->static_thread_mgmt01[0] = 0xffffffff;
4484 mqd->static_thread_mgmt01[1] = 0xffffffff;
4485 mqd->static_thread_mgmt23[0] = 0xffffffff;
4486 mqd->static_thread_mgmt23[1] = 0xffffffff;
4488 mutex_lock(&rdev->srbm_mutex);
4489 cik_srbm_select(rdev, rdev->ring[idx].me,
4490 rdev->ring[idx].pipe,
4491 rdev->ring[idx].queue, 0);
4493 /* disable wptr polling */
4494 tmp = RREG32(CP_PQ_WPTR_POLL_CNTL);
4495 tmp &= ~WPTR_POLL_EN;
4496 WREG32(CP_PQ_WPTR_POLL_CNTL, tmp);
4498 /* enable doorbell? */
4499 mqd->queue_state.cp_hqd_pq_doorbell_control =
4500 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4502 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4504 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_EN;
4505 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4506 mqd->queue_state.cp_hqd_pq_doorbell_control);
4508 /* disable the queue if it's active */
4509 mqd->queue_state.cp_hqd_dequeue_request = 0;
4510 mqd->queue_state.cp_hqd_pq_rptr = 0;
4511 mqd->queue_state.cp_hqd_pq_wptr= 0;
4512 if (RREG32(CP_HQD_ACTIVE) & 1) {
4513 WREG32(CP_HQD_DEQUEUE_REQUEST, 1);
4514 for (i = 0; i < rdev->usec_timeout; i++) {
4515 if (!(RREG32(CP_HQD_ACTIVE) & 1))
4519 WREG32(CP_HQD_DEQUEUE_REQUEST, mqd->queue_state.cp_hqd_dequeue_request);
4520 WREG32(CP_HQD_PQ_RPTR, mqd->queue_state.cp_hqd_pq_rptr);
4521 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4524 /* set the pointer to the MQD */
4525 mqd->queue_state.cp_mqd_base_addr = mqd_gpu_addr & 0xfffffffc;
4526 mqd->queue_state.cp_mqd_base_addr_hi = upper_32_bits(mqd_gpu_addr);
4527 WREG32(CP_MQD_BASE_ADDR, mqd->queue_state.cp_mqd_base_addr);
4528 WREG32(CP_MQD_BASE_ADDR_HI, mqd->queue_state.cp_mqd_base_addr_hi);
4529 /* set MQD vmid to 0 */
4530 mqd->queue_state.cp_mqd_control = RREG32(CP_MQD_CONTROL);
4531 mqd->queue_state.cp_mqd_control &= ~MQD_VMID_MASK;
4532 WREG32(CP_MQD_CONTROL, mqd->queue_state.cp_mqd_control);
4534 /* set the pointer to the HQD, this is similar CP_RB0_BASE/_HI */
4535 hqd_gpu_addr = rdev->ring[idx].gpu_addr >> 8;
4536 mqd->queue_state.cp_hqd_pq_base = hqd_gpu_addr;
4537 mqd->queue_state.cp_hqd_pq_base_hi = upper_32_bits(hqd_gpu_addr);
4538 WREG32(CP_HQD_PQ_BASE, mqd->queue_state.cp_hqd_pq_base);
4539 WREG32(CP_HQD_PQ_BASE_HI, mqd->queue_state.cp_hqd_pq_base_hi);
4541 /* set up the HQD, this is similar to CP_RB0_CNTL */
4542 mqd->queue_state.cp_hqd_pq_control = RREG32(CP_HQD_PQ_CONTROL);
4543 mqd->queue_state.cp_hqd_pq_control &=
4544 ~(QUEUE_SIZE_MASK | RPTR_BLOCK_SIZE_MASK);
4546 mqd->queue_state.cp_hqd_pq_control |=
4547 order_base_2(rdev->ring[idx].ring_size / 8);
4548 mqd->queue_state.cp_hqd_pq_control |=
4549 (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8);
4551 mqd->queue_state.cp_hqd_pq_control |= BUF_SWAP_32BIT;
4553 mqd->queue_state.cp_hqd_pq_control &=
4554 ~(UNORD_DISPATCH | ROQ_PQ_IB_FLIP | PQ_VOLATILE);
4555 mqd->queue_state.cp_hqd_pq_control |=
4556 PRIV_STATE | KMD_QUEUE; /* assuming kernel queue control */
4557 WREG32(CP_HQD_PQ_CONTROL, mqd->queue_state.cp_hqd_pq_control);
4559 /* only used if CP_PQ_WPTR_POLL_CNTL.WPTR_POLL_EN=1 */
4561 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP1_WPTR_OFFSET;
4563 wb_gpu_addr = rdev->wb.gpu_addr + CIK_WB_CP2_WPTR_OFFSET;
4564 mqd->queue_state.cp_hqd_pq_wptr_poll_addr = wb_gpu_addr & 0xfffffffc;
4565 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi = upper_32_bits(wb_gpu_addr) & 0xffff;
4566 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR, mqd->queue_state.cp_hqd_pq_wptr_poll_addr);
4567 WREG32(CP_HQD_PQ_WPTR_POLL_ADDR_HI,
4568 mqd->queue_state.cp_hqd_pq_wptr_poll_addr_hi);
4570 /* set the wb address wether it's enabled or not */
4572 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET;
4574 wb_gpu_addr = rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET;
4575 mqd->queue_state.cp_hqd_pq_rptr_report_addr = wb_gpu_addr & 0xfffffffc;
4576 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi =
4577 upper_32_bits(wb_gpu_addr) & 0xffff;
4578 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR,
4579 mqd->queue_state.cp_hqd_pq_rptr_report_addr);
4580 WREG32(CP_HQD_PQ_RPTR_REPORT_ADDR_HI,
4581 mqd->queue_state.cp_hqd_pq_rptr_report_addr_hi);
4583 /* enable the doorbell if requested */
4585 mqd->queue_state.cp_hqd_pq_doorbell_control =
4586 RREG32(CP_HQD_PQ_DOORBELL_CONTROL);
4587 mqd->queue_state.cp_hqd_pq_doorbell_control &= ~DOORBELL_OFFSET_MASK;
4588 mqd->queue_state.cp_hqd_pq_doorbell_control |=
4589 DOORBELL_OFFSET(rdev->ring[idx].doorbell_index);
4590 mqd->queue_state.cp_hqd_pq_doorbell_control |= DOORBELL_EN;
4591 mqd->queue_state.cp_hqd_pq_doorbell_control &=
4592 ~(DOORBELL_SOURCE | DOORBELL_HIT);
4595 mqd->queue_state.cp_hqd_pq_doorbell_control = 0;
4597 WREG32(CP_HQD_PQ_DOORBELL_CONTROL,
4598 mqd->queue_state.cp_hqd_pq_doorbell_control);
4600 /* read and write pointers, similar to CP_RB0_WPTR/_RPTR */
4601 rdev->ring[idx].wptr = 0;
4602 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
4603 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
4604 rdev->ring[idx].rptr = RREG32(CP_HQD_PQ_RPTR);
4605 mqd->queue_state.cp_hqd_pq_rptr = rdev->ring[idx].rptr;
4607 /* set the vmid for the queue */
4608 mqd->queue_state.cp_hqd_vmid = 0;
4609 WREG32(CP_HQD_VMID, mqd->queue_state.cp_hqd_vmid);
4611 /* activate the queue */
4612 mqd->queue_state.cp_hqd_active = 1;
4613 WREG32(CP_HQD_ACTIVE, mqd->queue_state.cp_hqd_active);
4615 cik_srbm_select(rdev, 0, 0, 0, 0);
4616 mutex_unlock(&rdev->srbm_mutex);
4618 radeon_bo_kunmap(rdev->ring[idx].mqd_obj);
4619 radeon_bo_unreserve(rdev->ring[idx].mqd_obj);
4621 rdev->ring[idx].ready = true;
4622 r = radeon_ring_test(rdev, idx, &rdev->ring[idx]);
4624 rdev->ring[idx].ready = false;
4630 static void cik_cp_enable(struct radeon_device *rdev, bool enable)
4632 cik_cp_gfx_enable(rdev, enable);
4633 cik_cp_compute_enable(rdev, enable);
4636 static int cik_cp_load_microcode(struct radeon_device *rdev)
4640 r = cik_cp_gfx_load_microcode(rdev);
4643 r = cik_cp_compute_load_microcode(rdev);
4650 static void cik_cp_fini(struct radeon_device *rdev)
4652 cik_cp_gfx_fini(rdev);
4653 cik_cp_compute_fini(rdev);
4656 static int cik_cp_resume(struct radeon_device *rdev)
4660 cik_enable_gui_idle_interrupt(rdev, false);
4662 r = cik_cp_load_microcode(rdev);
4666 r = cik_cp_gfx_resume(rdev);
4669 r = cik_cp_compute_resume(rdev);
4673 cik_enable_gui_idle_interrupt(rdev, true);
4678 static void cik_print_gpu_status_regs(struct radeon_device *rdev)
4680 dev_info(rdev->dev, " GRBM_STATUS=0x%08X\n",
4681 RREG32(GRBM_STATUS));
4682 dev_info(rdev->dev, " GRBM_STATUS2=0x%08X\n",
4683 RREG32(GRBM_STATUS2));
4684 dev_info(rdev->dev, " GRBM_STATUS_SE0=0x%08X\n",
4685 RREG32(GRBM_STATUS_SE0));
4686 dev_info(rdev->dev, " GRBM_STATUS_SE1=0x%08X\n",
4687 RREG32(GRBM_STATUS_SE1));
4688 dev_info(rdev->dev, " GRBM_STATUS_SE2=0x%08X\n",
4689 RREG32(GRBM_STATUS_SE2));
4690 dev_info(rdev->dev, " GRBM_STATUS_SE3=0x%08X\n",
4691 RREG32(GRBM_STATUS_SE3));
4692 dev_info(rdev->dev, " SRBM_STATUS=0x%08X\n",
4693 RREG32(SRBM_STATUS));
4694 dev_info(rdev->dev, " SRBM_STATUS2=0x%08X\n",
4695 RREG32(SRBM_STATUS2));
4696 dev_info(rdev->dev, " SDMA0_STATUS_REG = 0x%08X\n",
4697 RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET));
4698 dev_info(rdev->dev, " SDMA1_STATUS_REG = 0x%08X\n",
4699 RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET));
4700 dev_info(rdev->dev, " CP_STAT = 0x%08x\n", RREG32(CP_STAT));
4701 dev_info(rdev->dev, " CP_STALLED_STAT1 = 0x%08x\n",
4702 RREG32(CP_STALLED_STAT1));
4703 dev_info(rdev->dev, " CP_STALLED_STAT2 = 0x%08x\n",
4704 RREG32(CP_STALLED_STAT2));
4705 dev_info(rdev->dev, " CP_STALLED_STAT3 = 0x%08x\n",
4706 RREG32(CP_STALLED_STAT3));
4707 dev_info(rdev->dev, " CP_CPF_BUSY_STAT = 0x%08x\n",
4708 RREG32(CP_CPF_BUSY_STAT));
4709 dev_info(rdev->dev, " CP_CPF_STALLED_STAT1 = 0x%08x\n",
4710 RREG32(CP_CPF_STALLED_STAT1));
4711 dev_info(rdev->dev, " CP_CPF_STATUS = 0x%08x\n", RREG32(CP_CPF_STATUS));
4712 dev_info(rdev->dev, " CP_CPC_BUSY_STAT = 0x%08x\n", RREG32(CP_CPC_BUSY_STAT));
4713 dev_info(rdev->dev, " CP_CPC_STALLED_STAT1 = 0x%08x\n",
4714 RREG32(CP_CPC_STALLED_STAT1));
4715 dev_info(rdev->dev, " CP_CPC_STATUS = 0x%08x\n", RREG32(CP_CPC_STATUS));
4719 * cik_gpu_check_soft_reset - check which blocks are busy
4721 * @rdev: radeon_device pointer
4723 * Check which blocks are busy and return the relevant reset
4724 * mask to be used by cik_gpu_soft_reset().
4725 * Returns a mask of the blocks to be reset.
4727 u32 cik_gpu_check_soft_reset(struct radeon_device *rdev)
4733 tmp = RREG32(GRBM_STATUS);
4734 if (tmp & (PA_BUSY | SC_BUSY |
4735 BCI_BUSY | SX_BUSY |
4736 TA_BUSY | VGT_BUSY |
4738 GDS_BUSY | SPI_BUSY |
4739 IA_BUSY | IA_BUSY_NO_DMA))
4740 reset_mask |= RADEON_RESET_GFX;
4742 if (tmp & (CP_BUSY | CP_COHERENCY_BUSY))
4743 reset_mask |= RADEON_RESET_CP;
4746 tmp = RREG32(GRBM_STATUS2);
4748 reset_mask |= RADEON_RESET_RLC;
4750 /* SDMA0_STATUS_REG */
4751 tmp = RREG32(SDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET);
4752 if (!(tmp & SDMA_IDLE))
4753 reset_mask |= RADEON_RESET_DMA;
4755 /* SDMA1_STATUS_REG */
4756 tmp = RREG32(SDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET);
4757 if (!(tmp & SDMA_IDLE))
4758 reset_mask |= RADEON_RESET_DMA1;
4761 tmp = RREG32(SRBM_STATUS2);
4762 if (tmp & SDMA_BUSY)
4763 reset_mask |= RADEON_RESET_DMA;
4765 if (tmp & SDMA1_BUSY)
4766 reset_mask |= RADEON_RESET_DMA1;
4769 tmp = RREG32(SRBM_STATUS);
4772 reset_mask |= RADEON_RESET_IH;
4775 reset_mask |= RADEON_RESET_SEM;
4777 if (tmp & GRBM_RQ_PENDING)
4778 reset_mask |= RADEON_RESET_GRBM;
4781 reset_mask |= RADEON_RESET_VMC;
4783 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
4784 MCC_BUSY | MCD_BUSY))
4785 reset_mask |= RADEON_RESET_MC;
4787 if (evergreen_is_display_hung(rdev))
4788 reset_mask |= RADEON_RESET_DISPLAY;
4790 /* Skip MC reset as it's mostly likely not hung, just busy */
4791 if (reset_mask & RADEON_RESET_MC) {
4792 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
4793 reset_mask &= ~RADEON_RESET_MC;
4800 * cik_gpu_soft_reset - soft reset GPU
4802 * @rdev: radeon_device pointer
4803 * @reset_mask: mask of which blocks to reset
4805 * Soft reset the blocks specified in @reset_mask.
4807 static void cik_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
4809 struct evergreen_mc_save save;
4810 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
4813 if (reset_mask == 0)
4816 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
4818 cik_print_gpu_status_regs(rdev);
4819 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4820 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4821 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4822 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
4831 /* Disable GFX parsing/prefetching */
4832 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4834 /* Disable MEC parsing/prefetching */
4835 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
4837 if (reset_mask & RADEON_RESET_DMA) {
4839 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
4841 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
4843 if (reset_mask & RADEON_RESET_DMA1) {
4845 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
4847 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
4850 evergreen_mc_stop(rdev, &save);
4851 if (evergreen_mc_wait_for_idle(rdev)) {
4852 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4855 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP))
4856 grbm_soft_reset = SOFT_RESET_CP | SOFT_RESET_GFX;
4858 if (reset_mask & RADEON_RESET_CP) {
4859 grbm_soft_reset |= SOFT_RESET_CP;
4861 srbm_soft_reset |= SOFT_RESET_GRBM;
4864 if (reset_mask & RADEON_RESET_DMA)
4865 srbm_soft_reset |= SOFT_RESET_SDMA;
4867 if (reset_mask & RADEON_RESET_DMA1)
4868 srbm_soft_reset |= SOFT_RESET_SDMA1;
4870 if (reset_mask & RADEON_RESET_DISPLAY)
4871 srbm_soft_reset |= SOFT_RESET_DC;
4873 if (reset_mask & RADEON_RESET_RLC)
4874 grbm_soft_reset |= SOFT_RESET_RLC;
4876 if (reset_mask & RADEON_RESET_SEM)
4877 srbm_soft_reset |= SOFT_RESET_SEM;
4879 if (reset_mask & RADEON_RESET_IH)
4880 srbm_soft_reset |= SOFT_RESET_IH;
4882 if (reset_mask & RADEON_RESET_GRBM)
4883 srbm_soft_reset |= SOFT_RESET_GRBM;
4885 if (reset_mask & RADEON_RESET_VMC)
4886 srbm_soft_reset |= SOFT_RESET_VMC;
4888 if (!(rdev->flags & RADEON_IS_IGP)) {
4889 if (reset_mask & RADEON_RESET_MC)
4890 srbm_soft_reset |= SOFT_RESET_MC;
4893 if (grbm_soft_reset) {
4894 tmp = RREG32(GRBM_SOFT_RESET);
4895 tmp |= grbm_soft_reset;
4896 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
4897 WREG32(GRBM_SOFT_RESET, tmp);
4898 tmp = RREG32(GRBM_SOFT_RESET);
4902 tmp &= ~grbm_soft_reset;
4903 WREG32(GRBM_SOFT_RESET, tmp);
4904 tmp = RREG32(GRBM_SOFT_RESET);
4907 if (srbm_soft_reset) {
4908 tmp = RREG32(SRBM_SOFT_RESET);
4909 tmp |= srbm_soft_reset;
4910 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
4911 WREG32(SRBM_SOFT_RESET, tmp);
4912 tmp = RREG32(SRBM_SOFT_RESET);
4916 tmp &= ~srbm_soft_reset;
4917 WREG32(SRBM_SOFT_RESET, tmp);
4918 tmp = RREG32(SRBM_SOFT_RESET);
4921 /* Wait a little for things to settle down */
4924 evergreen_mc_resume(rdev, &save);
4927 cik_print_gpu_status_regs(rdev);
4930 struct kv_reset_save_regs {
4931 u32 gmcon_reng_execute;
4936 static void kv_save_regs_for_reset(struct radeon_device *rdev,
4937 struct kv_reset_save_regs *save)
4939 save->gmcon_reng_execute = RREG32(GMCON_RENG_EXECUTE);
4940 save->gmcon_misc = RREG32(GMCON_MISC);
4941 save->gmcon_misc3 = RREG32(GMCON_MISC3);
4943 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute & ~RENG_EXECUTE_ON_PWR_UP);
4944 WREG32(GMCON_MISC, save->gmcon_misc & ~(RENG_EXECUTE_ON_REG_UPDATE |
4945 STCTRL_STUTTER_EN));
4948 static void kv_restore_regs_for_reset(struct radeon_device *rdev,
4949 struct kv_reset_save_regs *save)
4953 WREG32(GMCON_PGFSM_WRITE, 0);
4954 WREG32(GMCON_PGFSM_CONFIG, 0x200010ff);
4956 for (i = 0; i < 5; i++)
4957 WREG32(GMCON_PGFSM_WRITE, 0);
4959 WREG32(GMCON_PGFSM_WRITE, 0);
4960 WREG32(GMCON_PGFSM_CONFIG, 0x300010ff);
4962 for (i = 0; i < 5; i++)
4963 WREG32(GMCON_PGFSM_WRITE, 0);
4965 WREG32(GMCON_PGFSM_WRITE, 0x210000);
4966 WREG32(GMCON_PGFSM_CONFIG, 0xa00010ff);
4968 for (i = 0; i < 5; i++)
4969 WREG32(GMCON_PGFSM_WRITE, 0);
4971 WREG32(GMCON_PGFSM_WRITE, 0x21003);
4972 WREG32(GMCON_PGFSM_CONFIG, 0xb00010ff);
4974 for (i = 0; i < 5; i++)
4975 WREG32(GMCON_PGFSM_WRITE, 0);
4977 WREG32(GMCON_PGFSM_WRITE, 0x2b00);
4978 WREG32(GMCON_PGFSM_CONFIG, 0xc00010ff);
4980 for (i = 0; i < 5; i++)
4981 WREG32(GMCON_PGFSM_WRITE, 0);
4983 WREG32(GMCON_PGFSM_WRITE, 0);
4984 WREG32(GMCON_PGFSM_CONFIG, 0xd00010ff);
4986 for (i = 0; i < 5; i++)
4987 WREG32(GMCON_PGFSM_WRITE, 0);
4989 WREG32(GMCON_PGFSM_WRITE, 0x420000);
4990 WREG32(GMCON_PGFSM_CONFIG, 0x100010ff);
4992 for (i = 0; i < 5; i++)
4993 WREG32(GMCON_PGFSM_WRITE, 0);
4995 WREG32(GMCON_PGFSM_WRITE, 0x120202);
4996 WREG32(GMCON_PGFSM_CONFIG, 0x500010ff);
4998 for (i = 0; i < 5; i++)
4999 WREG32(GMCON_PGFSM_WRITE, 0);
5001 WREG32(GMCON_PGFSM_WRITE, 0x3e3e36);
5002 WREG32(GMCON_PGFSM_CONFIG, 0x600010ff);
5004 for (i = 0; i < 5; i++)
5005 WREG32(GMCON_PGFSM_WRITE, 0);
5007 WREG32(GMCON_PGFSM_WRITE, 0x373f3e);
5008 WREG32(GMCON_PGFSM_CONFIG, 0x700010ff);
5010 for (i = 0; i < 5; i++)
5011 WREG32(GMCON_PGFSM_WRITE, 0);
5013 WREG32(GMCON_PGFSM_WRITE, 0x3e1332);
5014 WREG32(GMCON_PGFSM_CONFIG, 0xe00010ff);
5016 WREG32(GMCON_MISC3, save->gmcon_misc3);
5017 WREG32(GMCON_MISC, save->gmcon_misc);
5018 WREG32(GMCON_RENG_EXECUTE, save->gmcon_reng_execute);
5021 static void cik_gpu_pci_config_reset(struct radeon_device *rdev)
5023 struct evergreen_mc_save save;
5024 struct kv_reset_save_regs kv_save = { 0 };
5027 dev_info(rdev->dev, "GPU pci config reset\n");
5035 /* Disable GFX parsing/prefetching */
5036 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
5038 /* Disable MEC parsing/prefetching */
5039 WREG32(CP_MEC_CNTL, MEC_ME1_HALT | MEC_ME2_HALT);
5042 tmp = RREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET);
5044 WREG32(SDMA0_ME_CNTL + SDMA0_REGISTER_OFFSET, tmp);
5046 tmp = RREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET);
5048 WREG32(SDMA0_ME_CNTL + SDMA1_REGISTER_OFFSET, tmp);
5049 /* XXX other engines? */
5051 /* halt the rlc, disable cp internal ints */
5056 /* disable mem access */
5057 evergreen_mc_stop(rdev, &save);
5058 if (evergreen_mc_wait_for_idle(rdev)) {
5059 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
5062 if (rdev->flags & RADEON_IS_IGP)
5063 kv_save_regs_for_reset(rdev, &kv_save);
5066 pci_clear_master(rdev->pdev);
5068 radeon_pci_config_reset(rdev);
5072 /* wait for asic to come out of reset */
5073 for (i = 0; i < rdev->usec_timeout; i++) {
5074 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
5079 /* does asic init need to be run first??? */
5080 if (rdev->flags & RADEON_IS_IGP)
5081 kv_restore_regs_for_reset(rdev, &kv_save);
5085 * cik_asic_reset - soft reset GPU
5087 * @rdev: radeon_device pointer
5089 * Look up which blocks are hung and attempt
5091 * Returns 0 for success.
5093 int cik_asic_reset(struct radeon_device *rdev)
5097 reset_mask = cik_gpu_check_soft_reset(rdev);
5100 r600_set_bios_scratch_engine_hung(rdev, true);
5102 /* try soft reset */
5103 cik_gpu_soft_reset(rdev, reset_mask);
5105 reset_mask = cik_gpu_check_soft_reset(rdev);
5107 /* try pci config reset */
5108 if (reset_mask && radeon_hard_reset)
5109 cik_gpu_pci_config_reset(rdev);
5111 reset_mask = cik_gpu_check_soft_reset(rdev);
5114 r600_set_bios_scratch_engine_hung(rdev, false);
5120 * cik_gfx_is_lockup - check if the 3D engine is locked up
5122 * @rdev: radeon_device pointer
5123 * @ring: radeon_ring structure holding ring information
5125 * Check if the 3D engine is locked up (CIK).
5126 * Returns true if the engine is locked, false if not.
5128 bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
5130 u32 reset_mask = cik_gpu_check_soft_reset(rdev);
5132 if (!(reset_mask & (RADEON_RESET_GFX |
5133 RADEON_RESET_COMPUTE |
5134 RADEON_RESET_CP))) {
5135 radeon_ring_lockup_update(ring);
5138 /* force CP activities */
5139 radeon_ring_force_activity(rdev, ring);
5140 return radeon_ring_test_lockup(rdev, ring);
5145 * cik_mc_program - program the GPU memory controller
5147 * @rdev: radeon_device pointer
5149 * Set the location of vram, gart, and AGP in the GPU's
5150 * physical address space (CIK).
5152 static void cik_mc_program(struct radeon_device *rdev)
5154 struct evergreen_mc_save save;
5158 /* Initialize HDP */
5159 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
5160 WREG32((0x2c14 + j), 0x00000000);
5161 WREG32((0x2c18 + j), 0x00000000);
5162 WREG32((0x2c1c + j), 0x00000000);
5163 WREG32((0x2c20 + j), 0x00000000);
5164 WREG32((0x2c24 + j), 0x00000000);
5166 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
5168 evergreen_mc_stop(rdev, &save);
5169 if (radeon_mc_wait_for_idle(rdev)) {
5170 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5172 /* Lockout access through VGA aperture*/
5173 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
5174 /* Update configuration */
5175 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
5176 rdev->mc.vram_start >> 12);
5177 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
5178 rdev->mc.vram_end >> 12);
5179 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
5180 rdev->vram_scratch.gpu_addr >> 12);
5181 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
5182 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
5183 WREG32(MC_VM_FB_LOCATION, tmp);
5184 /* XXX double check these! */
5185 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
5186 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
5187 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
5188 WREG32(MC_VM_AGP_BASE, 0);
5189 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
5190 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
5191 if (radeon_mc_wait_for_idle(rdev)) {
5192 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
5194 evergreen_mc_resume(rdev, &save);
5195 /* we need to own VRAM, so turn off the VGA renderer here
5196 * to stop it overwriting our objects */
5197 rv515_vga_render_disable(rdev);
5201 * cik_mc_init - initialize the memory controller driver params
5203 * @rdev: radeon_device pointer
5205 * Look up the amount of vram, vram width, and decide how to place
5206 * vram and gart within the GPU's physical address space (CIK).
5207 * Returns 0 for success.
5209 static int cik_mc_init(struct radeon_device *rdev)
5212 int chansize, numchan;
5214 /* Get VRAM informations */
5215 rdev->mc.vram_is_ddr = true;
5216 tmp = RREG32(MC_ARB_RAMCFG);
5217 if (tmp & CHANSIZE_MASK) {
5222 tmp = RREG32(MC_SHARED_CHMAP);
5223 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
5253 rdev->mc.vram_width = numchan * chansize;
5254 /* Could aper size report 0 ? */
5255 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
5256 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
5257 /* size in MB on si */
5258 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5259 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
5260 rdev->mc.visible_vram_size = rdev->mc.aper_size;
5261 si_vram_gtt_location(rdev, &rdev->mc);
5262 radeon_update_bandwidth_info(rdev);
5269 * VMID 0 is the physical GPU addresses as used by the kernel.
5270 * VMIDs 1-15 are used for userspace clients and are handled
5271 * by the radeon vm/hsa code.
5274 * cik_pcie_gart_tlb_flush - gart tlb flush callback
5276 * @rdev: radeon_device pointer
5278 * Flush the TLB for the VMID 0 page table (CIK).
5280 void cik_pcie_gart_tlb_flush(struct radeon_device *rdev)
5282 /* flush hdp cache */
5283 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0);
5285 /* bits 0-15 are the VM contexts0-15 */
5286 WREG32(VM_INVALIDATE_REQUEST, 0x1);
5290 * cik_pcie_gart_enable - gart enable
5292 * @rdev: radeon_device pointer
5294 * This sets up the TLBs, programs the page tables for VMID0,
5295 * sets up the hw for VMIDs 1-15 which are allocated on
5296 * demand, and sets up the global locations for the LDS, GDS,
5297 * and GPUVM for FSA64 clients (CIK).
5298 * Returns 0 for success, errors for failure.
5300 static int cik_pcie_gart_enable(struct radeon_device *rdev)
5304 if (rdev->gart.robj == NULL) {
5305 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
5308 r = radeon_gart_table_vram_pin(rdev);
5311 radeon_gart_restore(rdev);
5312 /* Setup TLB control */
5313 WREG32(MC_VM_MX_L1_TLB_CNTL,
5316 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5317 ENABLE_ADVANCED_DRIVER_MODEL |
5318 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5319 /* Setup L2 cache */
5320 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
5321 ENABLE_L2_FRAGMENT_PROCESSING |
5322 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5323 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5324 EFFECTIVE_L2_QUEUE_SIZE(7) |
5325 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5326 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
5327 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5328 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5329 /* setup context0 */
5330 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
5331 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
5332 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
5333 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
5334 (u32)(rdev->dummy_page.addr >> 12));
5335 WREG32(VM_CONTEXT0_CNTL2, 0);
5336 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
5337 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
5343 /* empty context1-15 */
5344 /* FIXME start with 4G, once using 2 level pt switch to full
5347 /* set vm size, must be a multiple of 4 */
5348 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
5349 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
5350 for (i = 1; i < 16; i++) {
5352 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
5353 rdev->gart.table_addr >> 12);
5355 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
5356 rdev->gart.table_addr >> 12);
5359 /* enable context1-15 */
5360 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
5361 (u32)(rdev->dummy_page.addr >> 12));
5362 WREG32(VM_CONTEXT1_CNTL2, 4);
5363 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
5364 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5365 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5366 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5367 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
5368 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
5369 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
5370 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
5371 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
5372 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
5373 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
5374 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
5375 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
5377 if (rdev->family == CHIP_KAVERI) {
5378 u32 tmp = RREG32(CHUB_CONTROL);
5380 WREG32(CHUB_CONTROL, tmp);
5383 /* XXX SH_MEM regs */
5384 /* where to put LDS, scratch, GPUVM in FSA64 space */
5385 mutex_lock(&rdev->srbm_mutex);
5386 for (i = 0; i < 16; i++) {
5387 cik_srbm_select(rdev, 0, 0, 0, i);
5388 /* CP and shaders */
5389 WREG32(SH_MEM_CONFIG, 0);
5390 WREG32(SH_MEM_APE1_BASE, 1);
5391 WREG32(SH_MEM_APE1_LIMIT, 0);
5392 WREG32(SH_MEM_BASES, 0);
5394 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA0_REGISTER_OFFSET, 0);
5395 WREG32(SDMA0_GFX_APE1_CNTL + SDMA0_REGISTER_OFFSET, 0);
5396 WREG32(SDMA0_GFX_VIRTUAL_ADDR + SDMA1_REGISTER_OFFSET, 0);
5397 WREG32(SDMA0_GFX_APE1_CNTL + SDMA1_REGISTER_OFFSET, 0);
5398 /* XXX SDMA RLC - todo */
5400 cik_srbm_select(rdev, 0, 0, 0, 0);
5401 mutex_unlock(&rdev->srbm_mutex);
5403 cik_pcie_gart_tlb_flush(rdev);
5404 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
5405 (unsigned)(rdev->mc.gtt_size >> 20),
5406 (unsigned long long)rdev->gart.table_addr);
5407 rdev->gart.ready = true;
5412 * cik_pcie_gart_disable - gart disable
5414 * @rdev: radeon_device pointer
5416 * This disables all VM page table (CIK).
5418 static void cik_pcie_gart_disable(struct radeon_device *rdev)
5420 /* Disable all tables */
5421 WREG32(VM_CONTEXT0_CNTL, 0);
5422 WREG32(VM_CONTEXT1_CNTL, 0);
5423 /* Setup TLB control */
5424 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
5425 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
5426 /* Setup L2 cache */
5428 ENABLE_L2_FRAGMENT_PROCESSING |
5429 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
5430 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
5431 EFFECTIVE_L2_QUEUE_SIZE(7) |
5432 CONTEXT1_IDENTITY_ACCESS_MODE(1));
5433 WREG32(VM_L2_CNTL2, 0);
5434 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
5435 L2_CACHE_BIGK_FRAGMENT_SIZE(6));
5436 radeon_gart_table_vram_unpin(rdev);
5440 * cik_pcie_gart_fini - vm fini callback
5442 * @rdev: radeon_device pointer
5444 * Tears down the driver GART/VM setup (CIK).
5446 static void cik_pcie_gart_fini(struct radeon_device *rdev)
5448 cik_pcie_gart_disable(rdev);
5449 radeon_gart_table_vram_free(rdev);
5450 radeon_gart_fini(rdev);
5455 * cik_ib_parse - vm ib_parse callback
5457 * @rdev: radeon_device pointer
5458 * @ib: indirect buffer pointer
5460 * CIK uses hw IB checking so this is a nop (CIK).
5462 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
5469 * VMID 0 is the physical GPU addresses as used by the kernel.
5470 * VMIDs 1-15 are used for userspace clients and are handled
5471 * by the radeon vm/hsa code.
5474 * cik_vm_init - cik vm init callback
5476 * @rdev: radeon_device pointer
5478 * Inits cik specific vm parameters (number of VMs, base of vram for
5479 * VMIDs 1-15) (CIK).
5480 * Returns 0 for success.
5482 int cik_vm_init(struct radeon_device *rdev)
5485 rdev->vm_manager.nvm = 16;
5486 /* base offset of vram pages */
5487 if (rdev->flags & RADEON_IS_IGP) {
5488 u64 tmp = RREG32(MC_VM_FB_OFFSET);
5490 rdev->vm_manager.vram_base_offset = tmp;
5492 rdev->vm_manager.vram_base_offset = 0;
5498 * cik_vm_fini - cik vm fini callback
5500 * @rdev: radeon_device pointer
5502 * Tear down any asic specific VM setup (CIK).
5504 void cik_vm_fini(struct radeon_device *rdev)
5509 * cik_vm_decode_fault - print human readable fault info
5511 * @rdev: radeon_device pointer
5512 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
5513 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
5515 * Print human readable fault information (CIK).
5517 static void cik_vm_decode_fault(struct radeon_device *rdev,
5518 u32 status, u32 addr, u32 mc_client)
5521 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
5522 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
5523 char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
5524 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
5526 if (rdev->family == CHIP_HAWAII)
5527 mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5529 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
5531 printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
5532 protections, vmid, addr,
5533 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
5534 block, mc_client, mc_id);
5538 * cik_vm_flush - cik vm flush using the CP
5540 * @rdev: radeon_device pointer
5542 * Update the page table base and flush the VM TLB
5543 * using the CP (CIK).
5545 void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
5547 struct radeon_ring *ring = &rdev->ring[ridx];
5548 int usepfp = (ridx == RADEON_RING_TYPE_GFX_INDEX);
5553 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5554 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(usepfp) |
5555 WRITE_DATA_DST_SEL(0)));
5557 radeon_ring_write(ring,
5558 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
5560 radeon_ring_write(ring,
5561 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
5563 radeon_ring_write(ring, 0);
5564 radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
5566 /* update SH_MEM_* regs */
5567 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5568 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5569 WRITE_DATA_DST_SEL(0)));
5570 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5571 radeon_ring_write(ring, 0);
5572 radeon_ring_write(ring, VMID(vm->id));
5574 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 6));
5575 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5576 WRITE_DATA_DST_SEL(0)));
5577 radeon_ring_write(ring, SH_MEM_BASES >> 2);
5578 radeon_ring_write(ring, 0);
5580 radeon_ring_write(ring, 0); /* SH_MEM_BASES */
5581 radeon_ring_write(ring, 0); /* SH_MEM_CONFIG */
5582 radeon_ring_write(ring, 1); /* SH_MEM_APE1_BASE */
5583 radeon_ring_write(ring, 0); /* SH_MEM_APE1_LIMIT */
5585 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5586 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5587 WRITE_DATA_DST_SEL(0)));
5588 radeon_ring_write(ring, SRBM_GFX_CNTL >> 2);
5589 radeon_ring_write(ring, 0);
5590 radeon_ring_write(ring, VMID(0));
5593 cik_hdp_flush_cp_ring_emit(rdev, ridx);
5595 /* bits 0-15 are the VM contexts0-15 */
5596 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5597 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
5598 WRITE_DATA_DST_SEL(0)));
5599 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5600 radeon_ring_write(ring, 0);
5601 radeon_ring_write(ring, 1 << vm->id);
5603 /* compute doesn't have PFP */
5605 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5606 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5607 radeon_ring_write(ring, 0x0);
5613 * The RLC is a multi-purpose microengine that handles a
5614 * variety of functions, the most important of which is
5615 * the interrupt controller.
5617 static void cik_enable_gui_idle_interrupt(struct radeon_device *rdev,
5620 u32 tmp = RREG32(CP_INT_CNTL_RING0);
5623 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5625 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5626 WREG32(CP_INT_CNTL_RING0, tmp);
5629 static void cik_enable_lbpw(struct radeon_device *rdev, bool enable)
5633 tmp = RREG32(RLC_LB_CNTL);
5635 tmp |= LOAD_BALANCE_ENABLE;
5637 tmp &= ~LOAD_BALANCE_ENABLE;
5638 WREG32(RLC_LB_CNTL, tmp);
5641 static void cik_wait_for_rlc_serdes(struct radeon_device *rdev)
5646 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
5647 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
5648 cik_select_se_sh(rdev, i, j);
5649 for (k = 0; k < rdev->usec_timeout; k++) {
5650 if (RREG32(RLC_SERDES_CU_MASTER_BUSY) == 0)
5656 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5658 mask = SE_MASTER_BUSY_MASK | GC_MASTER_BUSY | TC0_MASTER_BUSY | TC1_MASTER_BUSY;
5659 for (k = 0; k < rdev->usec_timeout; k++) {
5660 if ((RREG32(RLC_SERDES_NONCU_MASTER_BUSY) & mask) == 0)
5666 static void cik_update_rlc(struct radeon_device *rdev, u32 rlc)
5670 tmp = RREG32(RLC_CNTL);
5672 WREG32(RLC_CNTL, rlc);
5675 static u32 cik_halt_rlc(struct radeon_device *rdev)
5679 orig = data = RREG32(RLC_CNTL);
5681 if (data & RLC_ENABLE) {
5684 data &= ~RLC_ENABLE;
5685 WREG32(RLC_CNTL, data);
5687 for (i = 0; i < rdev->usec_timeout; i++) {
5688 if ((RREG32(RLC_GPM_STAT) & RLC_GPM_BUSY) == 0)
5693 cik_wait_for_rlc_serdes(rdev);
5699 void cik_enter_rlc_safe_mode(struct radeon_device *rdev)
5703 tmp = REQ | MESSAGE(MSG_ENTER_RLC_SAFE_MODE);
5704 WREG32(RLC_GPR_REG2, tmp);
5706 mask = GFX_POWER_STATUS | GFX_CLOCK_STATUS;
5707 for (i = 0; i < rdev->usec_timeout; i++) {
5708 if ((RREG32(RLC_GPM_STAT) & mask) == mask)
5713 for (i = 0; i < rdev->usec_timeout; i++) {
5714 if ((RREG32(RLC_GPR_REG2) & REQ) == 0)
5720 void cik_exit_rlc_safe_mode(struct radeon_device *rdev)
5724 tmp = REQ | MESSAGE(MSG_EXIT_RLC_SAFE_MODE);
5725 WREG32(RLC_GPR_REG2, tmp);
5729 * cik_rlc_stop - stop the RLC ME
5731 * @rdev: radeon_device pointer
5733 * Halt the RLC ME (MicroEngine) (CIK).
5735 static void cik_rlc_stop(struct radeon_device *rdev)
5737 WREG32(RLC_CNTL, 0);
5739 cik_enable_gui_idle_interrupt(rdev, false);
5741 cik_wait_for_rlc_serdes(rdev);
5745 * cik_rlc_start - start the RLC ME
5747 * @rdev: radeon_device pointer
5749 * Unhalt the RLC ME (MicroEngine) (CIK).
5751 static void cik_rlc_start(struct radeon_device *rdev)
5753 WREG32(RLC_CNTL, RLC_ENABLE);
5755 cik_enable_gui_idle_interrupt(rdev, true);
5761 * cik_rlc_resume - setup the RLC hw
5763 * @rdev: radeon_device pointer
5765 * Initialize the RLC registers, load the ucode,
5766 * and start the RLC (CIK).
5767 * Returns 0 for success, -EINVAL if the ucode is not available.
5769 static int cik_rlc_resume(struct radeon_device *rdev)
5772 const __be32 *fw_data;
5777 switch (rdev->family) {
5781 size = BONAIRE_RLC_UCODE_SIZE;
5784 size = KV_RLC_UCODE_SIZE;
5787 size = KB_RLC_UCODE_SIZE;
5794 tmp = RREG32(RLC_CGCG_CGLS_CTRL) & 0xfffffffc;
5795 WREG32(RLC_CGCG_CGLS_CTRL, tmp);
5803 WREG32(RLC_LB_CNTR_INIT, 0);
5804 WREG32(RLC_LB_CNTR_MAX, 0x00008000);
5806 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5807 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5808 WREG32(RLC_LB_PARAMS, 0x00600408);
5809 WREG32(RLC_LB_CNTL, 0x80000004);
5811 WREG32(RLC_MC_CNTL, 0);
5812 WREG32(RLC_UCODE_CNTL, 0);
5814 fw_data = (const __be32 *)rdev->rlc_fw->data;
5815 WREG32(RLC_GPM_UCODE_ADDR, 0);
5816 for (i = 0; i < size; i++)
5817 WREG32(RLC_GPM_UCODE_DATA, be32_to_cpup(fw_data++));
5818 WREG32(RLC_GPM_UCODE_ADDR, 0);
5820 /* XXX - find out what chips support lbpw */
5821 cik_enable_lbpw(rdev, false);
5823 if (rdev->family == CHIP_BONAIRE)
5824 WREG32(RLC_DRIVER_DMA_STATUS, 0);
5826 cik_rlc_start(rdev);
5831 static void cik_enable_cgcg(struct radeon_device *rdev, bool enable)
5833 u32 data, orig, tmp, tmp2;
5835 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5837 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5838 cik_enable_gui_idle_interrupt(rdev, true);
5840 tmp = cik_halt_rlc(rdev);
5842 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5843 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5844 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5845 tmp2 = BPM_ADDR_MASK | CGCG_OVERRIDE_0 | CGLS_ENABLE;
5846 WREG32(RLC_SERDES_WR_CTRL, tmp2);
5848 cik_update_rlc(rdev, tmp);
5850 data |= CGCG_EN | CGLS_EN;
5852 cik_enable_gui_idle_interrupt(rdev, false);
5854 RREG32(CB_CGTT_SCLK_CTRL);
5855 RREG32(CB_CGTT_SCLK_CTRL);
5856 RREG32(CB_CGTT_SCLK_CTRL);
5857 RREG32(CB_CGTT_SCLK_CTRL);
5859 data &= ~(CGCG_EN | CGLS_EN);
5863 WREG32(RLC_CGCG_CGLS_CTRL, data);
5867 static void cik_enable_mgcg(struct radeon_device *rdev, bool enable)
5869 u32 data, orig, tmp = 0;
5871 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5872 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) {
5873 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5874 orig = data = RREG32(CP_MEM_SLP_CNTL);
5875 data |= CP_MEM_LS_EN;
5877 WREG32(CP_MEM_SLP_CNTL, data);
5881 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5884 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5886 tmp = cik_halt_rlc(rdev);
5888 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5889 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5890 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5891 data = BPM_ADDR_MASK | MGCG_OVERRIDE_0;
5892 WREG32(RLC_SERDES_WR_CTRL, data);
5894 cik_update_rlc(rdev, tmp);
5896 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS) {
5897 orig = data = RREG32(CGTS_SM_CTRL_REG);
5898 data &= ~SM_MODE_MASK;
5899 data |= SM_MODE(0x2);
5900 data |= SM_MODE_ENABLE;
5901 data &= ~CGTS_OVERRIDE;
5902 if ((rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGLS) &&
5903 (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGTS_LS))
5904 data &= ~CGTS_LS_OVERRIDE;
5905 data &= ~ON_MONITOR_ADD_MASK;
5906 data |= ON_MONITOR_ADD_EN;
5907 data |= ON_MONITOR_ADD(0x96);
5909 WREG32(CGTS_SM_CTRL_REG, data);
5912 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5915 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5917 data = RREG32(RLC_MEM_SLP_CNTL);
5918 if (data & RLC_MEM_LS_EN) {
5919 data &= ~RLC_MEM_LS_EN;
5920 WREG32(RLC_MEM_SLP_CNTL, data);
5923 data = RREG32(CP_MEM_SLP_CNTL);
5924 if (data & CP_MEM_LS_EN) {
5925 data &= ~CP_MEM_LS_EN;
5926 WREG32(CP_MEM_SLP_CNTL, data);
5929 orig = data = RREG32(CGTS_SM_CTRL_REG);
5930 data |= CGTS_OVERRIDE | CGTS_LS_OVERRIDE;
5932 WREG32(CGTS_SM_CTRL_REG, data);
5934 tmp = cik_halt_rlc(rdev);
5936 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5937 WREG32(RLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
5938 WREG32(RLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
5939 data = BPM_ADDR_MASK | MGCG_OVERRIDE_1;
5940 WREG32(RLC_SERDES_WR_CTRL, data);
5942 cik_update_rlc(rdev, tmp);
5946 static const u32 mc_cg_registers[] =
5959 static void cik_enable_mc_ls(struct radeon_device *rdev,
5965 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5966 orig = data = RREG32(mc_cg_registers[i]);
5967 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
5968 data |= MC_LS_ENABLE;
5970 data &= ~MC_LS_ENABLE;
5972 WREG32(mc_cg_registers[i], data);
5976 static void cik_enable_mc_mgcg(struct radeon_device *rdev,
5982 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5983 orig = data = RREG32(mc_cg_registers[i]);
5984 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
5985 data |= MC_CG_ENABLE;
5987 data &= ~MC_CG_ENABLE;
5989 WREG32(mc_cg_registers[i], data);
5993 static void cik_enable_sdma_mgcg(struct radeon_device *rdev,
5998 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
5999 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, 0x00000100);
6000 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, 0x00000100);
6002 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET);
6005 WREG32(SDMA0_CLK_CTRL + SDMA0_REGISTER_OFFSET, data);
6007 orig = data = RREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET);
6010 WREG32(SDMA0_CLK_CTRL + SDMA1_REGISTER_OFFSET, data);
6014 static void cik_enable_sdma_mgls(struct radeon_device *rdev,
6019 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_LS)) {
6020 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6023 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6025 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6028 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6030 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET);
6033 WREG32(SDMA0_POWER_CNTL + SDMA0_REGISTER_OFFSET, data);
6035 orig = data = RREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET);
6038 WREG32(SDMA0_POWER_CNTL + SDMA1_REGISTER_OFFSET, data);
6042 static void cik_enable_uvd_mgcg(struct radeon_device *rdev,
6047 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
6048 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6050 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6052 orig = data = RREG32(UVD_CGC_CTRL);
6055 WREG32(UVD_CGC_CTRL, data);
6057 data = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
6059 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, data);
6061 orig = data = RREG32(UVD_CGC_CTRL);
6064 WREG32(UVD_CGC_CTRL, data);
6068 static void cik_enable_bif_mgls(struct radeon_device *rdev,
6073 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
6075 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
6076 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
6077 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
6079 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
6080 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
6083 WREG32_PCIE_PORT(PCIE_CNTL2, data);
6086 static void cik_enable_hdp_mgcg(struct radeon_device *rdev,
6091 orig = data = RREG32(HDP_HOST_PATH_CNTL);
6093 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
6094 data &= ~CLOCK_GATING_DIS;
6096 data |= CLOCK_GATING_DIS;
6099 WREG32(HDP_HOST_PATH_CNTL, data);
6102 static void cik_enable_hdp_ls(struct radeon_device *rdev,
6107 orig = data = RREG32(HDP_MEM_POWER_LS);
6109 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
6110 data |= HDP_LS_ENABLE;
6112 data &= ~HDP_LS_ENABLE;
6115 WREG32(HDP_MEM_POWER_LS, data);
6118 void cik_update_cg(struct radeon_device *rdev,
6119 u32 block, bool enable)
6122 if (block & RADEON_CG_BLOCK_GFX) {
6123 cik_enable_gui_idle_interrupt(rdev, false);
6124 /* order matters! */
6126 cik_enable_mgcg(rdev, true);
6127 cik_enable_cgcg(rdev, true);
6129 cik_enable_cgcg(rdev, false);
6130 cik_enable_mgcg(rdev, false);
6132 cik_enable_gui_idle_interrupt(rdev, true);
6135 if (block & RADEON_CG_BLOCK_MC) {
6136 if (!(rdev->flags & RADEON_IS_IGP)) {
6137 cik_enable_mc_mgcg(rdev, enable);
6138 cik_enable_mc_ls(rdev, enable);
6142 if (block & RADEON_CG_BLOCK_SDMA) {
6143 cik_enable_sdma_mgcg(rdev, enable);
6144 cik_enable_sdma_mgls(rdev, enable);
6147 if (block & RADEON_CG_BLOCK_BIF) {
6148 cik_enable_bif_mgls(rdev, enable);
6151 if (block & RADEON_CG_BLOCK_UVD) {
6153 cik_enable_uvd_mgcg(rdev, enable);
6156 if (block & RADEON_CG_BLOCK_HDP) {
6157 cik_enable_hdp_mgcg(rdev, enable);
6158 cik_enable_hdp_ls(rdev, enable);
6162 static void cik_init_cg(struct radeon_device *rdev)
6165 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, true);
6168 si_init_uvd_internal_cg(rdev);
6170 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6171 RADEON_CG_BLOCK_SDMA |
6172 RADEON_CG_BLOCK_BIF |
6173 RADEON_CG_BLOCK_UVD |
6174 RADEON_CG_BLOCK_HDP), true);
6177 static void cik_fini_cg(struct radeon_device *rdev)
6179 cik_update_cg(rdev, (RADEON_CG_BLOCK_MC |
6180 RADEON_CG_BLOCK_SDMA |
6181 RADEON_CG_BLOCK_BIF |
6182 RADEON_CG_BLOCK_UVD |
6183 RADEON_CG_BLOCK_HDP), false);
6185 cik_update_cg(rdev, RADEON_CG_BLOCK_GFX, false);
6188 static void cik_enable_sck_slowdown_on_pu(struct radeon_device *rdev,
6193 orig = data = RREG32(RLC_PG_CNTL);
6194 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6195 data |= SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6197 data &= ~SMU_CLK_SLOWDOWN_ON_PU_ENABLE;
6199 WREG32(RLC_PG_CNTL, data);
6202 static void cik_enable_sck_slowdown_on_pd(struct radeon_device *rdev,
6207 orig = data = RREG32(RLC_PG_CNTL);
6208 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_RLC_SMU_HS))
6209 data |= SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6211 data &= ~SMU_CLK_SLOWDOWN_ON_PD_ENABLE;
6213 WREG32(RLC_PG_CNTL, data);
6216 static void cik_enable_cp_pg(struct radeon_device *rdev, bool enable)
6220 orig = data = RREG32(RLC_PG_CNTL);
6221 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_CP))
6222 data &= ~DISABLE_CP_PG;
6224 data |= DISABLE_CP_PG;
6226 WREG32(RLC_PG_CNTL, data);
6229 static void cik_enable_gds_pg(struct radeon_device *rdev, bool enable)
6233 orig = data = RREG32(RLC_PG_CNTL);
6234 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GDS))
6235 data &= ~DISABLE_GDS_PG;
6237 data |= DISABLE_GDS_PG;
6239 WREG32(RLC_PG_CNTL, data);
6242 #define CP_ME_TABLE_SIZE 96
6243 #define CP_ME_TABLE_OFFSET 2048
6244 #define CP_MEC_TABLE_OFFSET 4096
6246 void cik_init_cp_pg_table(struct radeon_device *rdev)
6248 const __be32 *fw_data;
6249 volatile u32 *dst_ptr;
6250 int me, i, max_me = 4;
6254 if (rdev->family == CHIP_KAVERI)
6257 if (rdev->rlc.cp_table_ptr == NULL)
6260 /* write the cp table buffer */
6261 dst_ptr = rdev->rlc.cp_table_ptr;
6262 for (me = 0; me < max_me; me++) {
6264 fw_data = (const __be32 *)rdev->ce_fw->data;
6265 table_offset = CP_ME_TABLE_OFFSET;
6266 } else if (me == 1) {
6267 fw_data = (const __be32 *)rdev->pfp_fw->data;
6268 table_offset = CP_ME_TABLE_OFFSET;
6269 } else if (me == 2) {
6270 fw_data = (const __be32 *)rdev->me_fw->data;
6271 table_offset = CP_ME_TABLE_OFFSET;
6273 fw_data = (const __be32 *)rdev->mec_fw->data;
6274 table_offset = CP_MEC_TABLE_OFFSET;
6277 for (i = 0; i < CP_ME_TABLE_SIZE; i ++) {
6278 dst_ptr[bo_offset + i] = cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
6280 bo_offset += CP_ME_TABLE_SIZE;
6284 static void cik_enable_gfx_cgpg(struct radeon_device *rdev,
6289 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
6290 orig = data = RREG32(RLC_PG_CNTL);
6291 data |= GFX_PG_ENABLE;
6293 WREG32(RLC_PG_CNTL, data);
6295 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6298 WREG32(RLC_AUTO_PG_CTRL, data);
6300 orig = data = RREG32(RLC_PG_CNTL);
6301 data &= ~GFX_PG_ENABLE;
6303 WREG32(RLC_PG_CNTL, data);
6305 orig = data = RREG32(RLC_AUTO_PG_CTRL);
6306 data &= ~AUTO_PG_EN;
6308 WREG32(RLC_AUTO_PG_CTRL, data);
6310 data = RREG32(DB_RENDER_CONTROL);
6314 static u32 cik_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
6316 u32 mask = 0, tmp, tmp1;
6319 cik_select_se_sh(rdev, se, sh);
6320 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
6321 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
6322 cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
6329 for (i = 0; i < rdev->config.cik.max_cu_per_sh; i ++) {
6334 return (~tmp) & mask;
6337 static void cik_init_ao_cu_mask(struct radeon_device *rdev)
6339 u32 i, j, k, active_cu_number = 0;
6340 u32 mask, counter, cu_bitmap;
6343 for (i = 0; i < rdev->config.cik.max_shader_engines; i++) {
6344 for (j = 0; j < rdev->config.cik.max_sh_per_se; j++) {
6348 for (k = 0; k < rdev->config.cik.max_cu_per_sh; k ++) {
6349 if (cik_get_cu_active_bitmap(rdev, i, j) & mask) {
6357 active_cu_number += counter;
6358 tmp |= (cu_bitmap << (i * 16 + j * 8));
6362 WREG32(RLC_PG_AO_CU_MASK, tmp);
6364 tmp = RREG32(RLC_MAX_PG_CU);
6365 tmp &= ~MAX_PU_CU_MASK;
6366 tmp |= MAX_PU_CU(active_cu_number);
6367 WREG32(RLC_MAX_PG_CU, tmp);
6370 static void cik_enable_gfx_static_mgpg(struct radeon_device *rdev,
6375 orig = data = RREG32(RLC_PG_CNTL);
6376 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_SMG))
6377 data |= STATIC_PER_CU_PG_ENABLE;
6379 data &= ~STATIC_PER_CU_PG_ENABLE;
6381 WREG32(RLC_PG_CNTL, data);
6384 static void cik_enable_gfx_dynamic_mgpg(struct radeon_device *rdev,
6389 orig = data = RREG32(RLC_PG_CNTL);
6390 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_DMG))
6391 data |= DYN_PER_CU_PG_ENABLE;
6393 data &= ~DYN_PER_CU_PG_ENABLE;
6395 WREG32(RLC_PG_CNTL, data);
6398 #define RLC_SAVE_AND_RESTORE_STARTING_OFFSET 0x90
6399 #define RLC_CLEAR_STATE_DESCRIPTOR_OFFSET 0x3D
6401 static void cik_init_gfx_cgpg(struct radeon_device *rdev)
6406 if (rdev->rlc.cs_data) {
6407 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6408 WREG32(RLC_GPM_SCRATCH_DATA, upper_32_bits(rdev->rlc.clear_state_gpu_addr));
6409 WREG32(RLC_GPM_SCRATCH_DATA, lower_32_bits(rdev->rlc.clear_state_gpu_addr));
6410 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.clear_state_size);
6412 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_CLEAR_STATE_DESCRIPTOR_OFFSET);
6413 for (i = 0; i < 3; i++)
6414 WREG32(RLC_GPM_SCRATCH_DATA, 0);
6416 if (rdev->rlc.reg_list) {
6417 WREG32(RLC_GPM_SCRATCH_ADDR, RLC_SAVE_AND_RESTORE_STARTING_OFFSET);
6418 for (i = 0; i < rdev->rlc.reg_list_size; i++)
6419 WREG32(RLC_GPM_SCRATCH_DATA, rdev->rlc.reg_list[i]);
6422 orig = data = RREG32(RLC_PG_CNTL);
6425 WREG32(RLC_PG_CNTL, data);
6427 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
6428 WREG32(RLC_CP_TABLE_RESTORE, rdev->rlc.cp_table_gpu_addr >> 8);
6430 data = RREG32(CP_RB_WPTR_POLL_CNTL);
6431 data &= ~IDLE_POLL_COUNT_MASK;
6432 data |= IDLE_POLL_COUNT(0x60);
6433 WREG32(CP_RB_WPTR_POLL_CNTL, data);
6436 WREG32(RLC_PG_DELAY, data);
6438 data = RREG32(RLC_PG_DELAY_2);
6441 WREG32(RLC_PG_DELAY_2, data);
6443 data = RREG32(RLC_AUTO_PG_CTRL);
6444 data &= ~GRBM_REG_SGIT_MASK;
6445 data |= GRBM_REG_SGIT(0x700);
6446 WREG32(RLC_AUTO_PG_CTRL, data);
6450 static void cik_update_gfx_pg(struct radeon_device *rdev, bool enable)
6452 cik_enable_gfx_cgpg(rdev, enable);
6453 cik_enable_gfx_static_mgpg(rdev, enable);
6454 cik_enable_gfx_dynamic_mgpg(rdev, enable);
6457 u32 cik_get_csb_size(struct radeon_device *rdev)
6460 const struct cs_section_def *sect = NULL;
6461 const struct cs_extent_def *ext = NULL;
6463 if (rdev->rlc.cs_data == NULL)
6466 /* begin clear state */
6468 /* context control state */
6471 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6472 for (ext = sect->section; ext->extent != NULL; ++ext) {
6473 if (sect->id == SECT_CONTEXT)
6474 count += 2 + ext->reg_count;
6479 /* pa_sc_raster_config/pa_sc_raster_config1 */
6481 /* end clear state */
6489 void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
6492 const struct cs_section_def *sect = NULL;
6493 const struct cs_extent_def *ext = NULL;
6495 if (rdev->rlc.cs_data == NULL)
6500 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6501 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
6503 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
6504 buffer[count++] = cpu_to_le32(0x80000000);
6505 buffer[count++] = cpu_to_le32(0x80000000);
6507 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
6508 for (ext = sect->section; ext->extent != NULL; ++ext) {
6509 if (sect->id == SECT_CONTEXT) {
6511 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
6512 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
6513 for (i = 0; i < ext->reg_count; i++)
6514 buffer[count++] = cpu_to_le32(ext->extent[i]);
6521 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
6522 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
6523 switch (rdev->family) {
6525 buffer[count++] = cpu_to_le32(0x16000012);
6526 buffer[count++] = cpu_to_le32(0x00000000);
6529 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6530 buffer[count++] = cpu_to_le32(0x00000000);
6533 buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
6534 buffer[count++] = cpu_to_le32(0x00000000);
6537 buffer[count++] = cpu_to_le32(0x3a00161a);
6538 buffer[count++] = cpu_to_le32(0x0000002e);
6541 buffer[count++] = cpu_to_le32(0x00000000);
6542 buffer[count++] = cpu_to_le32(0x00000000);
6546 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
6547 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
6549 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
6550 buffer[count++] = cpu_to_le32(0);
6553 static void cik_init_pg(struct radeon_device *rdev)
6555 if (rdev->pg_flags) {
6556 cik_enable_sck_slowdown_on_pu(rdev, true);
6557 cik_enable_sck_slowdown_on_pd(rdev, true);
6558 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
6559 cik_init_gfx_cgpg(rdev);
6560 cik_enable_cp_pg(rdev, true);
6561 cik_enable_gds_pg(rdev, true);
6563 cik_init_ao_cu_mask(rdev);
6564 cik_update_gfx_pg(rdev, true);
6568 static void cik_fini_pg(struct radeon_device *rdev)
6570 if (rdev->pg_flags) {
6571 cik_update_gfx_pg(rdev, false);
6572 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
6573 cik_enable_cp_pg(rdev, false);
6574 cik_enable_gds_pg(rdev, false);
6581 * Starting with r6xx, interrupts are handled via a ring buffer.
6582 * Ring buffers are areas of GPU accessible memory that the GPU
6583 * writes interrupt vectors into and the host reads vectors out of.
6584 * There is a rptr (read pointer) that determines where the
6585 * host is currently reading, and a wptr (write pointer)
6586 * which determines where the GPU has written. When the
6587 * pointers are equal, the ring is idle. When the GPU
6588 * writes vectors to the ring buffer, it increments the
6589 * wptr. When there is an interrupt, the host then starts
6590 * fetching commands and processing them until the pointers are
6591 * equal again at which point it updates the rptr.
6595 * cik_enable_interrupts - Enable the interrupt ring buffer
6597 * @rdev: radeon_device pointer
6599 * Enable the interrupt ring buffer (CIK).
6601 static void cik_enable_interrupts(struct radeon_device *rdev)
6603 u32 ih_cntl = RREG32(IH_CNTL);
6604 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6606 ih_cntl |= ENABLE_INTR;
6607 ih_rb_cntl |= IH_RB_ENABLE;
6608 WREG32(IH_CNTL, ih_cntl);
6609 WREG32(IH_RB_CNTL, ih_rb_cntl);
6610 rdev->ih.enabled = true;
6614 * cik_disable_interrupts - Disable the interrupt ring buffer
6616 * @rdev: radeon_device pointer
6618 * Disable the interrupt ring buffer (CIK).
6620 static void cik_disable_interrupts(struct radeon_device *rdev)
6622 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
6623 u32 ih_cntl = RREG32(IH_CNTL);
6625 ih_rb_cntl &= ~IH_RB_ENABLE;
6626 ih_cntl &= ~ENABLE_INTR;
6627 WREG32(IH_RB_CNTL, ih_rb_cntl);
6628 WREG32(IH_CNTL, ih_cntl);
6629 /* set rptr, wptr to 0 */
6630 WREG32(IH_RB_RPTR, 0);
6631 WREG32(IH_RB_WPTR, 0);
6632 rdev->ih.enabled = false;
6637 * cik_disable_interrupt_state - Disable all interrupt sources
6639 * @rdev: radeon_device pointer
6641 * Clear all interrupt enable bits used by the driver (CIK).
6643 static void cik_disable_interrupt_state(struct radeon_device *rdev)
6648 tmp = RREG32(CP_INT_CNTL_RING0) &
6649 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6650 WREG32(CP_INT_CNTL_RING0, tmp);
6652 tmp = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6653 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, tmp);
6654 tmp = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6655 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, tmp);
6656 /* compute queues */
6657 WREG32(CP_ME1_PIPE0_INT_CNTL, 0);
6658 WREG32(CP_ME1_PIPE1_INT_CNTL, 0);
6659 WREG32(CP_ME1_PIPE2_INT_CNTL, 0);
6660 WREG32(CP_ME1_PIPE3_INT_CNTL, 0);
6661 WREG32(CP_ME2_PIPE0_INT_CNTL, 0);
6662 WREG32(CP_ME2_PIPE1_INT_CNTL, 0);
6663 WREG32(CP_ME2_PIPE2_INT_CNTL, 0);
6664 WREG32(CP_ME2_PIPE3_INT_CNTL, 0);
6666 WREG32(GRBM_INT_CNTL, 0);
6667 /* vline/vblank, etc. */
6668 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6669 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6670 if (rdev->num_crtc >= 4) {
6671 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6672 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6674 if (rdev->num_crtc >= 6) {
6675 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6676 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6679 if (rdev->num_crtc >= 2) {
6680 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
6681 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
6683 if (rdev->num_crtc >= 4) {
6684 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
6685 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
6687 if (rdev->num_crtc >= 6) {
6688 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
6689 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
6693 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
6695 /* digital hotplug */
6696 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6697 WREG32(DC_HPD1_INT_CONTROL, tmp);
6698 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6699 WREG32(DC_HPD2_INT_CONTROL, tmp);
6700 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6701 WREG32(DC_HPD3_INT_CONTROL, tmp);
6702 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6703 WREG32(DC_HPD4_INT_CONTROL, tmp);
6704 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6705 WREG32(DC_HPD5_INT_CONTROL, tmp);
6706 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
6707 WREG32(DC_HPD6_INT_CONTROL, tmp);
6712 * cik_irq_init - init and enable the interrupt ring
6714 * @rdev: radeon_device pointer
6716 * Allocate a ring buffer for the interrupt controller,
6717 * enable the RLC, disable interrupts, enable the IH
6718 * ring buffer and enable it (CIK).
6719 * Called at device load and reume.
6720 * Returns 0 for success, errors for failure.
6722 static int cik_irq_init(struct radeon_device *rdev)
6726 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
6729 ret = r600_ih_ring_alloc(rdev);
6734 cik_disable_interrupts(rdev);
6737 ret = cik_rlc_resume(rdev);
6739 r600_ih_ring_fini(rdev);
6743 /* setup interrupt control */
6744 /* XXX this should actually be a bus address, not an MC address. same on older asics */
6745 WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
6746 interrupt_cntl = RREG32(INTERRUPT_CNTL);
6747 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
6748 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
6750 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
6751 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
6752 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
6753 WREG32(INTERRUPT_CNTL, interrupt_cntl);
6755 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
6756 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
6758 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
6759 IH_WPTR_OVERFLOW_CLEAR |
6762 if (rdev->wb.enabled)
6763 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
6765 /* set the writeback address whether it's enabled or not */
6766 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
6767 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6769 WREG32(IH_RB_CNTL, ih_rb_cntl);
6771 /* set rptr, wptr to 0 */
6772 WREG32(IH_RB_RPTR, 0);
6773 WREG32(IH_RB_WPTR, 0);
6775 /* Default settings for IH_CNTL (disabled at first) */
6776 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6777 /* RPTR_REARM only works if msi's are enabled */
6778 if (rdev->msi_enabled)
6779 ih_cntl |= RPTR_REARM;
6780 WREG32(IH_CNTL, ih_cntl);
6782 /* force the active interrupt state to all disabled */
6783 cik_disable_interrupt_state(rdev);
6785 pci_set_master(rdev->pdev);
6788 cik_enable_interrupts(rdev);
6794 * cik_irq_set - enable/disable interrupt sources
6796 * @rdev: radeon_device pointer
6798 * Enable interrupt sources on the GPU (vblanks, hpd,
6800 * Returns 0 for success, errors for failure.
6802 int cik_irq_set(struct radeon_device *rdev)
6805 u32 cp_m1p0, cp_m1p1, cp_m1p2, cp_m1p3;
6806 u32 cp_m2p0, cp_m2p1, cp_m2p2, cp_m2p3;
6807 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
6808 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
6809 u32 grbm_int_cntl = 0;
6810 u32 dma_cntl, dma_cntl1;
6813 if (!rdev->irq.installed) {
6814 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6817 /* don't enable anything if the ih is disabled */
6818 if (!rdev->ih.enabled) {
6819 cik_disable_interrupts(rdev);
6820 /* force the active interrupt state to all disabled */
6821 cik_disable_interrupt_state(rdev);
6825 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6826 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6827 cp_int_cntl |= PRIV_INSTR_INT_ENABLE | PRIV_REG_INT_ENABLE;
6829 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
6830 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
6831 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
6832 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
6833 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
6834 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
6836 dma_cntl = RREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6837 dma_cntl1 = RREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6839 cp_m1p0 = RREG32(CP_ME1_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6840 cp_m1p1 = RREG32(CP_ME1_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6841 cp_m1p2 = RREG32(CP_ME1_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6842 cp_m1p3 = RREG32(CP_ME1_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6843 cp_m2p0 = RREG32(CP_ME2_PIPE0_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6844 cp_m2p1 = RREG32(CP_ME2_PIPE1_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6845 cp_m2p2 = RREG32(CP_ME2_PIPE2_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6846 cp_m2p3 = RREG32(CP_ME2_PIPE3_INT_CNTL) & ~TIME_STAMP_INT_ENABLE;
6848 if (rdev->flags & RADEON_IS_IGP)
6849 thermal_int = RREG32_SMC(CG_THERMAL_INT_CTRL) &
6850 ~(THERM_INTH_MASK | THERM_INTL_MASK);
6852 thermal_int = RREG32_SMC(CG_THERMAL_INT) &
6853 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
6855 /* enable CP interrupts on all rings */
6856 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6857 DRM_DEBUG("cik_irq_set: sw int gfx\n");
6858 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6860 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6861 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6862 DRM_DEBUG("si_irq_set: sw int cp1\n");
6863 if (ring->me == 1) {
6864 switch (ring->pipe) {
6866 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6869 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6872 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6875 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6878 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6881 } else if (ring->me == 2) {
6882 switch (ring->pipe) {
6884 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6887 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6890 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6893 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6896 DRM_DEBUG("si_irq_set: sw int cp1 invalid pipe %d\n", ring->pipe);
6900 DRM_DEBUG("si_irq_set: sw int cp1 invalid me %d\n", ring->me);
6903 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
6904 struct radeon_ring *ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6905 DRM_DEBUG("si_irq_set: sw int cp2\n");
6906 if (ring->me == 1) {
6907 switch (ring->pipe) {
6909 cp_m1p0 |= TIME_STAMP_INT_ENABLE;
6912 cp_m1p1 |= TIME_STAMP_INT_ENABLE;
6915 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6918 cp_m1p2 |= TIME_STAMP_INT_ENABLE;
6921 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6924 } else if (ring->me == 2) {
6925 switch (ring->pipe) {
6927 cp_m2p0 |= TIME_STAMP_INT_ENABLE;
6930 cp_m2p1 |= TIME_STAMP_INT_ENABLE;
6933 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6936 cp_m2p2 |= TIME_STAMP_INT_ENABLE;
6939 DRM_DEBUG("si_irq_set: sw int cp2 invalid pipe %d\n", ring->pipe);
6943 DRM_DEBUG("si_irq_set: sw int cp2 invalid me %d\n", ring->me);
6947 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
6948 DRM_DEBUG("cik_irq_set: sw int dma\n");
6949 dma_cntl |= TRAP_ENABLE;
6952 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
6953 DRM_DEBUG("cik_irq_set: sw int dma1\n");
6954 dma_cntl1 |= TRAP_ENABLE;
6957 if (rdev->irq.crtc_vblank_int[0] ||
6958 atomic_read(&rdev->irq.pflip[0])) {
6959 DRM_DEBUG("cik_irq_set: vblank 0\n");
6960 crtc1 |= VBLANK_INTERRUPT_MASK;
6962 if (rdev->irq.crtc_vblank_int[1] ||
6963 atomic_read(&rdev->irq.pflip[1])) {
6964 DRM_DEBUG("cik_irq_set: vblank 1\n");
6965 crtc2 |= VBLANK_INTERRUPT_MASK;
6967 if (rdev->irq.crtc_vblank_int[2] ||
6968 atomic_read(&rdev->irq.pflip[2])) {
6969 DRM_DEBUG("cik_irq_set: vblank 2\n");
6970 crtc3 |= VBLANK_INTERRUPT_MASK;
6972 if (rdev->irq.crtc_vblank_int[3] ||
6973 atomic_read(&rdev->irq.pflip[3])) {
6974 DRM_DEBUG("cik_irq_set: vblank 3\n");
6975 crtc4 |= VBLANK_INTERRUPT_MASK;
6977 if (rdev->irq.crtc_vblank_int[4] ||
6978 atomic_read(&rdev->irq.pflip[4])) {
6979 DRM_DEBUG("cik_irq_set: vblank 4\n");
6980 crtc5 |= VBLANK_INTERRUPT_MASK;
6982 if (rdev->irq.crtc_vblank_int[5] ||
6983 atomic_read(&rdev->irq.pflip[5])) {
6984 DRM_DEBUG("cik_irq_set: vblank 5\n");
6985 crtc6 |= VBLANK_INTERRUPT_MASK;
6987 if (rdev->irq.hpd[0]) {
6988 DRM_DEBUG("cik_irq_set: hpd 1\n");
6989 hpd1 |= DC_HPDx_INT_EN;
6991 if (rdev->irq.hpd[1]) {
6992 DRM_DEBUG("cik_irq_set: hpd 2\n");
6993 hpd2 |= DC_HPDx_INT_EN;
6995 if (rdev->irq.hpd[2]) {
6996 DRM_DEBUG("cik_irq_set: hpd 3\n");
6997 hpd3 |= DC_HPDx_INT_EN;
6999 if (rdev->irq.hpd[3]) {
7000 DRM_DEBUG("cik_irq_set: hpd 4\n");
7001 hpd4 |= DC_HPDx_INT_EN;
7003 if (rdev->irq.hpd[4]) {
7004 DRM_DEBUG("cik_irq_set: hpd 5\n");
7005 hpd5 |= DC_HPDx_INT_EN;
7007 if (rdev->irq.hpd[5]) {
7008 DRM_DEBUG("cik_irq_set: hpd 6\n");
7009 hpd6 |= DC_HPDx_INT_EN;
7012 if (rdev->irq.dpm_thermal) {
7013 DRM_DEBUG("dpm thermal\n");
7014 if (rdev->flags & RADEON_IS_IGP)
7015 thermal_int |= THERM_INTH_MASK | THERM_INTL_MASK;
7017 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
7020 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
7022 WREG32(SDMA0_CNTL + SDMA0_REGISTER_OFFSET, dma_cntl);
7023 WREG32(SDMA0_CNTL + SDMA1_REGISTER_OFFSET, dma_cntl1);
7025 WREG32(CP_ME1_PIPE0_INT_CNTL, cp_m1p0);
7026 WREG32(CP_ME1_PIPE1_INT_CNTL, cp_m1p1);
7027 WREG32(CP_ME1_PIPE2_INT_CNTL, cp_m1p2);
7028 WREG32(CP_ME1_PIPE3_INT_CNTL, cp_m1p3);
7029 WREG32(CP_ME2_PIPE0_INT_CNTL, cp_m2p0);
7030 WREG32(CP_ME2_PIPE1_INT_CNTL, cp_m2p1);
7031 WREG32(CP_ME2_PIPE2_INT_CNTL, cp_m2p2);
7032 WREG32(CP_ME2_PIPE3_INT_CNTL, cp_m2p3);
7034 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
7036 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
7037 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
7038 if (rdev->num_crtc >= 4) {
7039 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
7040 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
7042 if (rdev->num_crtc >= 6) {
7043 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
7044 WREG32(LB_INTERRUPT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
7047 if (rdev->num_crtc >= 2) {
7048 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
7049 GRPH_PFLIP_INT_MASK);
7050 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
7051 GRPH_PFLIP_INT_MASK);
7053 if (rdev->num_crtc >= 4) {
7054 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
7055 GRPH_PFLIP_INT_MASK);
7056 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
7057 GRPH_PFLIP_INT_MASK);
7059 if (rdev->num_crtc >= 6) {
7060 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
7061 GRPH_PFLIP_INT_MASK);
7062 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
7063 GRPH_PFLIP_INT_MASK);
7066 WREG32(DC_HPD1_INT_CONTROL, hpd1);
7067 WREG32(DC_HPD2_INT_CONTROL, hpd2);
7068 WREG32(DC_HPD3_INT_CONTROL, hpd3);
7069 WREG32(DC_HPD4_INT_CONTROL, hpd4);
7070 WREG32(DC_HPD5_INT_CONTROL, hpd5);
7071 WREG32(DC_HPD6_INT_CONTROL, hpd6);
7073 if (rdev->flags & RADEON_IS_IGP)
7074 WREG32_SMC(CG_THERMAL_INT_CTRL, thermal_int);
7076 WREG32_SMC(CG_THERMAL_INT, thermal_int);
7082 * cik_irq_ack - ack interrupt sources
7084 * @rdev: radeon_device pointer
7086 * Ack interrupt sources on the GPU (vblanks, hpd,
7087 * etc.) (CIK). Certain interrupts sources are sw
7088 * generated and do not require an explicit ack.
7090 static inline void cik_irq_ack(struct radeon_device *rdev)
7094 rdev->irq.stat_regs.cik.disp_int = RREG32(DISP_INTERRUPT_STATUS);
7095 rdev->irq.stat_regs.cik.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
7096 rdev->irq.stat_regs.cik.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
7097 rdev->irq.stat_regs.cik.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
7098 rdev->irq.stat_regs.cik.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
7099 rdev->irq.stat_regs.cik.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
7100 rdev->irq.stat_regs.cik.disp_int_cont6 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE6);
7102 rdev->irq.stat_regs.cik.d1grph_int = RREG32(GRPH_INT_STATUS +
7103 EVERGREEN_CRTC0_REGISTER_OFFSET);
7104 rdev->irq.stat_regs.cik.d2grph_int = RREG32(GRPH_INT_STATUS +
7105 EVERGREEN_CRTC1_REGISTER_OFFSET);
7106 if (rdev->num_crtc >= 4) {
7107 rdev->irq.stat_regs.cik.d3grph_int = RREG32(GRPH_INT_STATUS +
7108 EVERGREEN_CRTC2_REGISTER_OFFSET);
7109 rdev->irq.stat_regs.cik.d4grph_int = RREG32(GRPH_INT_STATUS +
7110 EVERGREEN_CRTC3_REGISTER_OFFSET);
7112 if (rdev->num_crtc >= 6) {
7113 rdev->irq.stat_regs.cik.d5grph_int = RREG32(GRPH_INT_STATUS +
7114 EVERGREEN_CRTC4_REGISTER_OFFSET);
7115 rdev->irq.stat_regs.cik.d6grph_int = RREG32(GRPH_INT_STATUS +
7116 EVERGREEN_CRTC5_REGISTER_OFFSET);
7119 if (rdev->irq.stat_regs.cik.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
7120 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET,
7121 GRPH_PFLIP_INT_CLEAR);
7122 if (rdev->irq.stat_regs.cik.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
7123 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET,
7124 GRPH_PFLIP_INT_CLEAR);
7125 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT)
7126 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
7127 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT)
7128 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
7129 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
7130 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
7131 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT)
7132 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
7134 if (rdev->num_crtc >= 4) {
7135 if (rdev->irq.stat_regs.cik.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
7136 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET,
7137 GRPH_PFLIP_INT_CLEAR);
7138 if (rdev->irq.stat_regs.cik.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
7139 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET,
7140 GRPH_PFLIP_INT_CLEAR);
7141 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
7142 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
7143 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
7144 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
7145 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
7146 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
7147 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
7148 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
7151 if (rdev->num_crtc >= 6) {
7152 if (rdev->irq.stat_regs.cik.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
7153 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET,
7154 GRPH_PFLIP_INT_CLEAR);
7155 if (rdev->irq.stat_regs.cik.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
7156 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET,
7157 GRPH_PFLIP_INT_CLEAR);
7158 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
7159 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
7160 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
7161 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
7162 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
7163 WREG32(LB_VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
7164 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
7165 WREG32(LB_VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
7168 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7169 tmp = RREG32(DC_HPD1_INT_CONTROL);
7170 tmp |= DC_HPDx_INT_ACK;
7171 WREG32(DC_HPD1_INT_CONTROL, tmp);
7173 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7174 tmp = RREG32(DC_HPD2_INT_CONTROL);
7175 tmp |= DC_HPDx_INT_ACK;
7176 WREG32(DC_HPD2_INT_CONTROL, tmp);
7178 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7179 tmp = RREG32(DC_HPD3_INT_CONTROL);
7180 tmp |= DC_HPDx_INT_ACK;
7181 WREG32(DC_HPD3_INT_CONTROL, tmp);
7183 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7184 tmp = RREG32(DC_HPD4_INT_CONTROL);
7185 tmp |= DC_HPDx_INT_ACK;
7186 WREG32(DC_HPD4_INT_CONTROL, tmp);
7188 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7189 tmp = RREG32(DC_HPD5_INT_CONTROL);
7190 tmp |= DC_HPDx_INT_ACK;
7191 WREG32(DC_HPD5_INT_CONTROL, tmp);
7193 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7194 tmp = RREG32(DC_HPD5_INT_CONTROL);
7195 tmp |= DC_HPDx_INT_ACK;
7196 WREG32(DC_HPD6_INT_CONTROL, tmp);
7201 * cik_irq_disable - disable interrupts
7203 * @rdev: radeon_device pointer
7205 * Disable interrupts on the hw (CIK).
7207 static void cik_irq_disable(struct radeon_device *rdev)
7209 cik_disable_interrupts(rdev);
7210 /* Wait and acknowledge irq */
7213 cik_disable_interrupt_state(rdev);
7217 * cik_irq_disable - disable interrupts for suspend
7219 * @rdev: radeon_device pointer
7221 * Disable interrupts and stop the RLC (CIK).
7224 static void cik_irq_suspend(struct radeon_device *rdev)
7226 cik_irq_disable(rdev);
7231 * cik_irq_fini - tear down interrupt support
7233 * @rdev: radeon_device pointer
7235 * Disable interrupts on the hw and free the IH ring
7237 * Used for driver unload.
7239 static void cik_irq_fini(struct radeon_device *rdev)
7241 cik_irq_suspend(rdev);
7242 r600_ih_ring_fini(rdev);
7246 * cik_get_ih_wptr - get the IH ring buffer wptr
7248 * @rdev: radeon_device pointer
7250 * Get the IH ring buffer wptr from either the register
7251 * or the writeback memory buffer (CIK). Also check for
7252 * ring buffer overflow and deal with it.
7253 * Used by cik_irq_process().
7254 * Returns the value of the wptr.
7256 static inline u32 cik_get_ih_wptr(struct radeon_device *rdev)
7260 if (rdev->wb.enabled)
7261 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
7263 wptr = RREG32(IH_RB_WPTR);
7265 if (wptr & RB_OVERFLOW) {
7266 /* When a ring buffer overflow happen start parsing interrupt
7267 * from the last not overwritten vector (wptr + 16). Hopefully
7268 * this should allow us to catchup.
7270 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
7271 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
7272 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
7273 tmp = RREG32(IH_RB_CNTL);
7274 tmp |= IH_WPTR_OVERFLOW_CLEAR;
7275 WREG32(IH_RB_CNTL, tmp);
7276 wptr &= ~RB_OVERFLOW;
7278 return (wptr & rdev->ih.ptr_mask);
7282 * Each IV ring entry is 128 bits:
7283 * [7:0] - interrupt source id
7285 * [59:32] - interrupt source data
7286 * [63:60] - reserved
7289 * ME_ID [1:0], PIPE_ID[1:0], QUEUE_ID[2:0]
7290 * QUEUE_ID - for compute, which of the 8 queues owned by the dispatcher
7291 * - for gfx, hw shader state (0=PS...5=LS, 6=CS)
7292 * ME_ID - 0 = gfx, 1 = first 4 CS pipes, 2 = second 4 CS pipes
7293 * PIPE_ID - ME0 0=3D
7294 * - ME1&2 compute dispatcher (4 pipes each)
7296 * INSTANCE_ID [1:0], QUEUE_ID[1:0]
7297 * INSTANCE_ID - 0 = sdma0, 1 = sdma1
7298 * QUEUE_ID - 0 = gfx, 1 = rlc0, 2 = rlc1
7301 * [127:96] - reserved
7304 * cik_irq_process - interrupt handler
7306 * @rdev: radeon_device pointer
7308 * Interrupt hander (CIK). Walk the IH ring,
7309 * ack interrupts and schedule work to handle
7311 * Returns irq process return code.
7313 int cik_irq_process(struct radeon_device *rdev)
7315 struct radeon_ring *cp1_ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7316 struct radeon_ring *cp2_ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7319 u32 src_id, src_data, ring_id;
7320 u8 me_id, pipe_id, queue_id;
7322 bool queue_hotplug = false;
7323 bool queue_reset = false;
7324 u32 addr, status, mc_client;
7325 bool queue_thermal = false;
7327 if (!rdev->ih.enabled || rdev->shutdown)
7330 wptr = cik_get_ih_wptr(rdev);
7333 /* is somebody else already processing irqs? */
7334 if (atomic_xchg(&rdev->ih.lock, 1))
7337 rptr = rdev->ih.rptr;
7338 DRM_DEBUG("cik_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
7340 /* Order reading of wptr vs. reading of IH ring data */
7343 /* display interrupts */
7346 while (rptr != wptr) {
7347 /* wptr/rptr are in bytes! */
7348 ring_index = rptr / 4;
7349 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
7350 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
7351 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
7354 case 1: /* D1 vblank/vline */
7356 case 0: /* D1 vblank */
7357 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VBLANK_INTERRUPT) {
7358 if (rdev->irq.crtc_vblank_int[0]) {
7359 drm_handle_vblank(rdev->ddev, 0);
7360 rdev->pm.vblank_sync = true;
7361 wake_up(&rdev->irq.vblank_queue);
7363 if (atomic_read(&rdev->irq.pflip[0]))
7364 radeon_crtc_handle_flip(rdev, 0);
7365 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
7366 DRM_DEBUG("IH: D1 vblank\n");
7369 case 1: /* D1 vline */
7370 if (rdev->irq.stat_regs.cik.disp_int & LB_D1_VLINE_INTERRUPT) {
7371 rdev->irq.stat_regs.cik.disp_int &= ~LB_D1_VLINE_INTERRUPT;
7372 DRM_DEBUG("IH: D1 vline\n");
7376 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7380 case 2: /* D2 vblank/vline */
7382 case 0: /* D2 vblank */
7383 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
7384 if (rdev->irq.crtc_vblank_int[1]) {
7385 drm_handle_vblank(rdev->ddev, 1);
7386 rdev->pm.vblank_sync = true;
7387 wake_up(&rdev->irq.vblank_queue);
7389 if (atomic_read(&rdev->irq.pflip[1]))
7390 radeon_crtc_handle_flip(rdev, 1);
7391 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
7392 DRM_DEBUG("IH: D2 vblank\n");
7395 case 1: /* D2 vline */
7396 if (rdev->irq.stat_regs.cik.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
7397 rdev->irq.stat_regs.cik.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
7398 DRM_DEBUG("IH: D2 vline\n");
7402 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7406 case 3: /* D3 vblank/vline */
7408 case 0: /* D3 vblank */
7409 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
7410 if (rdev->irq.crtc_vblank_int[2]) {
7411 drm_handle_vblank(rdev->ddev, 2);
7412 rdev->pm.vblank_sync = true;
7413 wake_up(&rdev->irq.vblank_queue);
7415 if (atomic_read(&rdev->irq.pflip[2]))
7416 radeon_crtc_handle_flip(rdev, 2);
7417 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
7418 DRM_DEBUG("IH: D3 vblank\n");
7421 case 1: /* D3 vline */
7422 if (rdev->irq.stat_regs.cik.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
7423 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
7424 DRM_DEBUG("IH: D3 vline\n");
7428 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7432 case 4: /* D4 vblank/vline */
7434 case 0: /* D4 vblank */
7435 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
7436 if (rdev->irq.crtc_vblank_int[3]) {
7437 drm_handle_vblank(rdev->ddev, 3);
7438 rdev->pm.vblank_sync = true;
7439 wake_up(&rdev->irq.vblank_queue);
7441 if (atomic_read(&rdev->irq.pflip[3]))
7442 radeon_crtc_handle_flip(rdev, 3);
7443 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
7444 DRM_DEBUG("IH: D4 vblank\n");
7447 case 1: /* D4 vline */
7448 if (rdev->irq.stat_regs.cik.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
7449 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
7450 DRM_DEBUG("IH: D4 vline\n");
7454 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7458 case 5: /* D5 vblank/vline */
7460 case 0: /* D5 vblank */
7461 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
7462 if (rdev->irq.crtc_vblank_int[4]) {
7463 drm_handle_vblank(rdev->ddev, 4);
7464 rdev->pm.vblank_sync = true;
7465 wake_up(&rdev->irq.vblank_queue);
7467 if (atomic_read(&rdev->irq.pflip[4]))
7468 radeon_crtc_handle_flip(rdev, 4);
7469 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
7470 DRM_DEBUG("IH: D5 vblank\n");
7473 case 1: /* D5 vline */
7474 if (rdev->irq.stat_regs.cik.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
7475 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
7476 DRM_DEBUG("IH: D5 vline\n");
7480 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7484 case 6: /* D6 vblank/vline */
7486 case 0: /* D6 vblank */
7487 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
7488 if (rdev->irq.crtc_vblank_int[5]) {
7489 drm_handle_vblank(rdev->ddev, 5);
7490 rdev->pm.vblank_sync = true;
7491 wake_up(&rdev->irq.vblank_queue);
7493 if (atomic_read(&rdev->irq.pflip[5]))
7494 radeon_crtc_handle_flip(rdev, 5);
7495 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
7496 DRM_DEBUG("IH: D6 vblank\n");
7499 case 1: /* D6 vline */
7500 if (rdev->irq.stat_regs.cik.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
7501 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
7502 DRM_DEBUG("IH: D6 vline\n");
7506 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7510 case 8: /* D1 page flip */
7511 case 10: /* D2 page flip */
7512 case 12: /* D3 page flip */
7513 case 14: /* D4 page flip */
7514 case 16: /* D5 page flip */
7515 case 18: /* D6 page flip */
7516 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
7517 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
7519 case 42: /* HPD hotplug */
7522 if (rdev->irq.stat_regs.cik.disp_int & DC_HPD1_INTERRUPT) {
7523 rdev->irq.stat_regs.cik.disp_int &= ~DC_HPD1_INTERRUPT;
7524 queue_hotplug = true;
7525 DRM_DEBUG("IH: HPD1\n");
7529 if (rdev->irq.stat_regs.cik.disp_int_cont & DC_HPD2_INTERRUPT) {
7530 rdev->irq.stat_regs.cik.disp_int_cont &= ~DC_HPD2_INTERRUPT;
7531 queue_hotplug = true;
7532 DRM_DEBUG("IH: HPD2\n");
7536 if (rdev->irq.stat_regs.cik.disp_int_cont2 & DC_HPD3_INTERRUPT) {
7537 rdev->irq.stat_regs.cik.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
7538 queue_hotplug = true;
7539 DRM_DEBUG("IH: HPD3\n");
7543 if (rdev->irq.stat_regs.cik.disp_int_cont3 & DC_HPD4_INTERRUPT) {
7544 rdev->irq.stat_regs.cik.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
7545 queue_hotplug = true;
7546 DRM_DEBUG("IH: HPD4\n");
7550 if (rdev->irq.stat_regs.cik.disp_int_cont4 & DC_HPD5_INTERRUPT) {
7551 rdev->irq.stat_regs.cik.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
7552 queue_hotplug = true;
7553 DRM_DEBUG("IH: HPD5\n");
7557 if (rdev->irq.stat_regs.cik.disp_int_cont5 & DC_HPD6_INTERRUPT) {
7558 rdev->irq.stat_regs.cik.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
7559 queue_hotplug = true;
7560 DRM_DEBUG("IH: HPD6\n");
7564 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7569 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
7570 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
7574 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
7575 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
7576 mc_client = RREG32(VM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
7577 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
7578 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
7580 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
7582 cik_vm_decode_fault(rdev, status, addr, mc_client);
7583 /* reset addr and status */
7584 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
7586 case 176: /* GFX RB CP_INT */
7587 case 177: /* GFX IB CP_INT */
7588 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7590 case 181: /* CP EOP event */
7591 DRM_DEBUG("IH: CP EOP\n");
7592 /* XXX check the bitfield order! */
7593 me_id = (ring_id & 0x60) >> 5;
7594 pipe_id = (ring_id & 0x18) >> 3;
7595 queue_id = (ring_id & 0x7) >> 0;
7598 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
7602 if ((cp1_ring->me == me_id) & (cp1_ring->pipe == pipe_id))
7603 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7604 if ((cp2_ring->me == me_id) & (cp2_ring->pipe == pipe_id))
7605 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7609 case 184: /* CP Privileged reg access */
7610 DRM_ERROR("Illegal register access in command stream\n");
7611 /* XXX check the bitfield order! */
7612 me_id = (ring_id & 0x60) >> 5;
7613 pipe_id = (ring_id & 0x18) >> 3;
7614 queue_id = (ring_id & 0x7) >> 0;
7617 /* This results in a full GPU reset, but all we need to do is soft
7618 * reset the CP for gfx
7632 case 185: /* CP Privileged inst */
7633 DRM_ERROR("Illegal instruction in command stream\n");
7634 /* XXX check the bitfield order! */
7635 me_id = (ring_id & 0x60) >> 5;
7636 pipe_id = (ring_id & 0x18) >> 3;
7637 queue_id = (ring_id & 0x7) >> 0;
7640 /* This results in a full GPU reset, but all we need to do is soft
7641 * reset the CP for gfx
7655 case 224: /* SDMA trap event */
7656 /* XXX check the bitfield order! */
7657 me_id = (ring_id & 0x3) >> 0;
7658 queue_id = (ring_id & 0xc) >> 2;
7659 DRM_DEBUG("IH: SDMA trap\n");
7664 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
7677 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7689 case 230: /* thermal low to high */
7690 DRM_DEBUG("IH: thermal low to high\n");
7691 rdev->pm.dpm.thermal.high_to_low = false;
7692 queue_thermal = true;
7694 case 231: /* thermal high to low */
7695 DRM_DEBUG("IH: thermal high to low\n");
7696 rdev->pm.dpm.thermal.high_to_low = true;
7697 queue_thermal = true;
7699 case 233: /* GUI IDLE */
7700 DRM_DEBUG("IH: GUI idle\n");
7702 case 241: /* SDMA Privileged inst */
7703 case 247: /* SDMA Privileged inst */
7704 DRM_ERROR("Illegal instruction in SDMA command stream\n");
7705 /* XXX check the bitfield order! */
7706 me_id = (ring_id & 0x3) >> 0;
7707 queue_id = (ring_id & 0xc) >> 2;
7742 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
7746 /* wptr/rptr are in bytes! */
7748 rptr &= rdev->ih.ptr_mask;
7751 schedule_work(&rdev->hotplug_work);
7753 schedule_work(&rdev->reset_work);
7755 schedule_work(&rdev->pm.dpm.thermal.work);
7756 rdev->ih.rptr = rptr;
7757 WREG32(IH_RB_RPTR, rdev->ih.rptr);
7758 atomic_set(&rdev->ih.lock, 0);
7760 /* make sure wptr hasn't changed while processing */
7761 wptr = cik_get_ih_wptr(rdev);
7769 * startup/shutdown callbacks
7772 * cik_startup - program the asic to a functional state
7774 * @rdev: radeon_device pointer
7776 * Programs the asic to a functional state (CIK).
7777 * Called by cik_init() and cik_resume().
7778 * Returns 0 for success, error for failure.
7780 static int cik_startup(struct radeon_device *rdev)
7782 struct radeon_ring *ring;
7786 /* enable pcie gen2/3 link */
7787 cik_pcie_gen3_enable(rdev);
7789 cik_program_aspm(rdev);
7791 /* scratch needs to be initialized before MC */
7792 r = r600_vram_scratch_init(rdev);
7796 cik_mc_program(rdev);
7798 if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
7799 r = ci_mc_load_microcode(rdev);
7801 DRM_ERROR("Failed to load MC firmware!\n");
7806 r = cik_pcie_gart_enable(rdev);
7811 /* allocate rlc buffers */
7812 if (rdev->flags & RADEON_IS_IGP) {
7813 if (rdev->family == CHIP_KAVERI) {
7814 rdev->rlc.reg_list = spectre_rlc_save_restore_register_list;
7815 rdev->rlc.reg_list_size =
7816 (u32)ARRAY_SIZE(spectre_rlc_save_restore_register_list);
7818 rdev->rlc.reg_list = kalindi_rlc_save_restore_register_list;
7819 rdev->rlc.reg_list_size =
7820 (u32)ARRAY_SIZE(kalindi_rlc_save_restore_register_list);
7823 rdev->rlc.cs_data = ci_cs_data;
7824 rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
7825 r = sumo_rlc_init(rdev);
7827 DRM_ERROR("Failed to init rlc BOs!\n");
7831 /* allocate wb buffer */
7832 r = radeon_wb_init(rdev);
7836 /* allocate mec buffers */
7837 r = cik_mec_init(rdev);
7839 DRM_ERROR("Failed to init MEC BOs!\n");
7843 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
7845 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7849 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
7851 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7855 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
7857 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
7861 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
7863 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7867 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
7869 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
7873 r = radeon_uvd_resume(rdev);
7875 r = uvd_v4_2_resume(rdev);
7877 r = radeon_fence_driver_start_ring(rdev,
7878 R600_RING_TYPE_UVD_INDEX);
7880 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
7884 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
7887 if (!rdev->irq.installed) {
7888 r = radeon_irq_kms_init(rdev);
7893 r = cik_irq_init(rdev);
7895 DRM_ERROR("radeon: IH init failed (%d).\n", r);
7896 radeon_irq_kms_fini(rdev);
7901 if (rdev->family == CHIP_HAWAII) {
7902 nop = RADEON_CP_PACKET2;
7904 nop = PACKET3(PACKET3_NOP, 0x3FFF);
7907 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7908 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
7913 /* set up the compute queues */
7914 /* type-2 packets are deprecated on MEC, use type-3 instead */
7915 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7916 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
7920 ring->me = 1; /* first MEC */
7921 ring->pipe = 0; /* first pipe */
7922 ring->queue = 0; /* first queue */
7923 ring->wptr_offs = CIK_WB_CP1_WPTR_OFFSET;
7925 /* type-2 packets are deprecated on MEC, use type-3 instead */
7926 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7927 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
7931 /* dGPU only have 1 MEC */
7932 ring->me = 1; /* first MEC */
7933 ring->pipe = 0; /* first pipe */
7934 ring->queue = 1; /* second queue */
7935 ring->wptr_offs = CIK_WB_CP2_WPTR_OFFSET;
7937 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7938 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
7939 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7943 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7944 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
7945 SDMA_PACKET(SDMA_OPCODE_NOP, 0, 0));
7949 r = cik_cp_resume(rdev);
7953 r = cik_sdma_resume(rdev);
7957 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7958 if (ring->ring_size) {
7959 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
7962 r = uvd_v1_0_init(rdev);
7964 DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
7967 r = radeon_ib_pool_init(rdev);
7969 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
7973 r = radeon_vm_manager_init(rdev);
7975 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
7979 r = dce6_audio_init(rdev);
7987 * cik_resume - resume the asic to a functional state
7989 * @rdev: radeon_device pointer
7991 * Programs the asic to a functional state (CIK).
7993 * Returns 0 for success, error for failure.
7995 int cik_resume(struct radeon_device *rdev)
8000 atom_asic_init(rdev->mode_info.atom_context);
8002 /* init golden registers */
8003 cik_init_golden_registers(rdev);
8005 if (rdev->pm.pm_method == PM_METHOD_DPM)
8006 radeon_pm_resume(rdev);
8008 rdev->accel_working = true;
8009 r = cik_startup(rdev);
8011 DRM_ERROR("cik startup failed on resume\n");
8012 rdev->accel_working = false;
8021 * cik_suspend - suspend the asic
8023 * @rdev: radeon_device pointer
8025 * Bring the chip into a state suitable for suspend (CIK).
8026 * Called at suspend.
8027 * Returns 0 for success.
8029 int cik_suspend(struct radeon_device *rdev)
8031 radeon_pm_suspend(rdev);
8032 dce6_audio_fini(rdev);
8033 radeon_vm_manager_fini(rdev);
8034 cik_cp_enable(rdev, false);
8035 cik_sdma_enable(rdev, false);
8036 uvd_v1_0_fini(rdev);
8037 radeon_uvd_suspend(rdev);
8040 cik_irq_suspend(rdev);
8041 radeon_wb_disable(rdev);
8042 cik_pcie_gart_disable(rdev);
8046 /* Plan is to move initialization in that function and use
8047 * helper function so that radeon_device_init pretty much
8048 * do nothing more than calling asic specific function. This
8049 * should also allow to remove a bunch of callback function
8053 * cik_init - asic specific driver and hw init
8055 * @rdev: radeon_device pointer
8057 * Setup asic specific driver variables and program the hw
8058 * to a functional state (CIK).
8059 * Called at driver startup.
8060 * Returns 0 for success, errors for failure.
8062 int cik_init(struct radeon_device *rdev)
8064 struct radeon_ring *ring;
8068 if (!radeon_get_bios(rdev)) {
8069 if (ASIC_IS_AVIVO(rdev))
8072 /* Must be an ATOMBIOS */
8073 if (!rdev->is_atom_bios) {
8074 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
8077 r = radeon_atombios_init(rdev);
8081 /* Post card if necessary */
8082 if (!radeon_card_posted(rdev)) {
8084 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
8087 DRM_INFO("GPU not posted. posting now...\n");
8088 atom_asic_init(rdev->mode_info.atom_context);
8090 /* init golden registers */
8091 cik_init_golden_registers(rdev);
8092 /* Initialize scratch registers */
8093 cik_scratch_init(rdev);
8094 /* Initialize surface registers */
8095 radeon_surface_init(rdev);
8096 /* Initialize clocks */
8097 radeon_get_clock_info(rdev->ddev);
8100 r = radeon_fence_driver_init(rdev);
8104 /* initialize memory controller */
8105 r = cik_mc_init(rdev);
8108 /* Memory manager */
8109 r = radeon_bo_init(rdev);
8113 if (rdev->flags & RADEON_IS_IGP) {
8114 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8115 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw) {
8116 r = cik_init_microcode(rdev);
8118 DRM_ERROR("Failed to load firmware!\n");
8123 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
8124 !rdev->mec_fw || !rdev->sdma_fw || !rdev->rlc_fw ||
8126 r = cik_init_microcode(rdev);
8128 DRM_ERROR("Failed to load firmware!\n");
8134 /* Initialize power management */
8135 radeon_pm_init(rdev);
8137 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
8138 ring->ring_obj = NULL;
8139 r600_ring_init(rdev, ring, 1024 * 1024);
8141 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
8142 ring->ring_obj = NULL;
8143 r600_ring_init(rdev, ring, 1024 * 1024);
8144 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8148 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
8149 ring->ring_obj = NULL;
8150 r600_ring_init(rdev, ring, 1024 * 1024);
8151 r = radeon_doorbell_get(rdev, &ring->doorbell_index);
8155 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
8156 ring->ring_obj = NULL;
8157 r600_ring_init(rdev, ring, 256 * 1024);
8159 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
8160 ring->ring_obj = NULL;
8161 r600_ring_init(rdev, ring, 256 * 1024);
8163 r = radeon_uvd_init(rdev);
8165 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
8166 ring->ring_obj = NULL;
8167 r600_ring_init(rdev, ring, 4096);
8170 rdev->ih.ring_obj = NULL;
8171 r600_ih_ring_init(rdev, 64 * 1024);
8173 r = r600_pcie_gart_init(rdev);
8177 rdev->accel_working = true;
8178 r = cik_startup(rdev);
8180 dev_err(rdev->dev, "disabling GPU acceleration\n");
8182 cik_sdma_fini(rdev);
8184 sumo_rlc_fini(rdev);
8186 radeon_wb_fini(rdev);
8187 radeon_ib_pool_fini(rdev);
8188 radeon_vm_manager_fini(rdev);
8189 radeon_irq_kms_fini(rdev);
8190 cik_pcie_gart_fini(rdev);
8191 rdev->accel_working = false;
8194 /* Don't start up if the MC ucode is missing.
8195 * The default clocks and voltages before the MC ucode
8196 * is loaded are not suffient for advanced operations.
8198 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
8199 DRM_ERROR("radeon: MC ucode required for NI+.\n");
8207 * cik_fini - asic specific driver and hw fini
8209 * @rdev: radeon_device pointer
8211 * Tear down the asic specific driver variables and program the hw
8212 * to an idle state (CIK).
8213 * Called at driver unload.
8215 void cik_fini(struct radeon_device *rdev)
8217 radeon_pm_fini(rdev);
8219 cik_sdma_fini(rdev);
8223 sumo_rlc_fini(rdev);
8225 radeon_wb_fini(rdev);
8226 radeon_vm_manager_fini(rdev);
8227 radeon_ib_pool_fini(rdev);
8228 radeon_irq_kms_fini(rdev);
8229 uvd_v1_0_fini(rdev);
8230 radeon_uvd_fini(rdev);
8231 cik_pcie_gart_fini(rdev);
8232 r600_vram_scratch_fini(rdev);
8233 radeon_gem_fini(rdev);
8234 radeon_fence_driver_fini(rdev);
8235 radeon_bo_fini(rdev);
8236 radeon_atombios_fini(rdev);
8241 void dce8_program_fmt(struct drm_encoder *encoder)
8243 struct drm_device *dev = encoder->dev;
8244 struct radeon_device *rdev = dev->dev_private;
8245 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
8246 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
8247 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
8250 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
8253 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
8254 bpc = radeon_get_monitor_bpc(connector);
8255 dither = radeon_connector->dither;
8258 /* LVDS/eDP FMT is set up by atom */
8259 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
8262 /* not needed for analog */
8263 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
8264 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
8272 if (dither == RADEON_FMT_DITHER_ENABLE)
8273 /* XXX sort out optimal dither settings */
8274 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8275 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
8277 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
8280 if (dither == RADEON_FMT_DITHER_ENABLE)
8281 /* XXX sort out optimal dither settings */
8282 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8283 FMT_RGB_RANDOM_ENABLE |
8284 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
8286 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
8289 if (dither == RADEON_FMT_DITHER_ENABLE)
8290 /* XXX sort out optimal dither settings */
8291 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
8292 FMT_RGB_RANDOM_ENABLE |
8293 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
8295 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
8302 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
8305 /* display watermark setup */
8307 * dce8_line_buffer_adjust - Set up the line buffer
8309 * @rdev: radeon_device pointer
8310 * @radeon_crtc: the selected display controller
8311 * @mode: the current display mode on the selected display
8314 * Setup up the line buffer allocation for
8315 * the selected display controller (CIK).
8316 * Returns the line buffer size in pixels.
8318 static u32 dce8_line_buffer_adjust(struct radeon_device *rdev,
8319 struct radeon_crtc *radeon_crtc,
8320 struct drm_display_mode *mode)
8322 u32 tmp, buffer_alloc, i;
8323 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
8326 * There are 6 line buffers, one for each display controllers.
8327 * There are 3 partitions per LB. Select the number of partitions
8328 * to enable based on the display width. For display widths larger
8329 * than 4096, you need use to use 2 display controllers and combine
8330 * them using the stereo blender.
8332 if (radeon_crtc->base.enabled && mode) {
8333 if (mode->crtc_hdisplay < 1920) {
8336 } else if (mode->crtc_hdisplay < 2560) {
8339 } else if (mode->crtc_hdisplay < 4096) {
8341 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8343 DRM_DEBUG_KMS("Mode too big for LB!\n");
8345 buffer_alloc = (rdev->flags & RADEON_IS_IGP) ? 2 : 4;
8352 WREG32(LB_MEMORY_CTRL + radeon_crtc->crtc_offset,
8353 LB_MEMORY_CONFIG(tmp) | LB_MEMORY_SIZE(0x6B0));
8355 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
8356 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
8357 for (i = 0; i < rdev->usec_timeout; i++) {
8358 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
8359 DMIF_BUFFERS_ALLOCATED_COMPLETED)
8364 if (radeon_crtc->base.enabled && mode) {
8376 /* controller not enabled, so no lb used */
8381 * cik_get_number_of_dram_channels - get the number of dram channels
8383 * @rdev: radeon_device pointer
8385 * Look up the number of video ram channels (CIK).
8386 * Used for display watermark bandwidth calculations
8387 * Returns the number of dram channels
8389 static u32 cik_get_number_of_dram_channels(struct radeon_device *rdev)
8391 u32 tmp = RREG32(MC_SHARED_CHMAP);
8393 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
8416 struct dce8_wm_params {
8417 u32 dram_channels; /* number of dram channels */
8418 u32 yclk; /* bandwidth per dram data pin in kHz */
8419 u32 sclk; /* engine clock in kHz */
8420 u32 disp_clk; /* display clock in kHz */
8421 u32 src_width; /* viewport width */
8422 u32 active_time; /* active display time in ns */
8423 u32 blank_time; /* blank time in ns */
8424 bool interlaced; /* mode is interlaced */
8425 fixed20_12 vsc; /* vertical scale ratio */
8426 u32 num_heads; /* number of active crtcs */
8427 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
8428 u32 lb_size; /* line buffer allocated to pipe */
8429 u32 vtaps; /* vertical scaler taps */
8433 * dce8_dram_bandwidth - get the dram bandwidth
8435 * @wm: watermark calculation data
8437 * Calculate the raw dram bandwidth (CIK).
8438 * Used for display watermark bandwidth calculations
8439 * Returns the dram bandwidth in MBytes/s
8441 static u32 dce8_dram_bandwidth(struct dce8_wm_params *wm)
8443 /* Calculate raw DRAM Bandwidth */
8444 fixed20_12 dram_efficiency; /* 0.7 */
8445 fixed20_12 yclk, dram_channels, bandwidth;
8448 a.full = dfixed_const(1000);
8449 yclk.full = dfixed_const(wm->yclk);
8450 yclk.full = dfixed_div(yclk, a);
8451 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8452 a.full = dfixed_const(10);
8453 dram_efficiency.full = dfixed_const(7);
8454 dram_efficiency.full = dfixed_div(dram_efficiency, a);
8455 bandwidth.full = dfixed_mul(dram_channels, yclk);
8456 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
8458 return dfixed_trunc(bandwidth);
8462 * dce8_dram_bandwidth_for_display - get the dram bandwidth for display
8464 * @wm: watermark calculation data
8466 * Calculate the dram bandwidth used for display (CIK).
8467 * Used for display watermark bandwidth calculations
8468 * Returns the dram bandwidth for display in MBytes/s
8470 static u32 dce8_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8472 /* Calculate DRAM Bandwidth and the part allocated to display. */
8473 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
8474 fixed20_12 yclk, dram_channels, bandwidth;
8477 a.full = dfixed_const(1000);
8478 yclk.full = dfixed_const(wm->yclk);
8479 yclk.full = dfixed_div(yclk, a);
8480 dram_channels.full = dfixed_const(wm->dram_channels * 4);
8481 a.full = dfixed_const(10);
8482 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
8483 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
8484 bandwidth.full = dfixed_mul(dram_channels, yclk);
8485 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
8487 return dfixed_trunc(bandwidth);
8491 * dce8_data_return_bandwidth - get the data return bandwidth
8493 * @wm: watermark calculation data
8495 * Calculate the data return bandwidth used for display (CIK).
8496 * Used for display watermark bandwidth calculations
8497 * Returns the data return bandwidth in MBytes/s
8499 static u32 dce8_data_return_bandwidth(struct dce8_wm_params *wm)
8501 /* Calculate the display Data return Bandwidth */
8502 fixed20_12 return_efficiency; /* 0.8 */
8503 fixed20_12 sclk, bandwidth;
8506 a.full = dfixed_const(1000);
8507 sclk.full = dfixed_const(wm->sclk);
8508 sclk.full = dfixed_div(sclk, a);
8509 a.full = dfixed_const(10);
8510 return_efficiency.full = dfixed_const(8);
8511 return_efficiency.full = dfixed_div(return_efficiency, a);
8512 a.full = dfixed_const(32);
8513 bandwidth.full = dfixed_mul(a, sclk);
8514 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
8516 return dfixed_trunc(bandwidth);
8520 * dce8_dmif_request_bandwidth - get the dmif bandwidth
8522 * @wm: watermark calculation data
8524 * Calculate the dmif bandwidth used for display (CIK).
8525 * Used for display watermark bandwidth calculations
8526 * Returns the dmif bandwidth in MBytes/s
8528 static u32 dce8_dmif_request_bandwidth(struct dce8_wm_params *wm)
8530 /* Calculate the DMIF Request Bandwidth */
8531 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
8532 fixed20_12 disp_clk, bandwidth;
8535 a.full = dfixed_const(1000);
8536 disp_clk.full = dfixed_const(wm->disp_clk);
8537 disp_clk.full = dfixed_div(disp_clk, a);
8538 a.full = dfixed_const(32);
8539 b.full = dfixed_mul(a, disp_clk);
8541 a.full = dfixed_const(10);
8542 disp_clk_request_efficiency.full = dfixed_const(8);
8543 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
8545 bandwidth.full = dfixed_mul(b, disp_clk_request_efficiency);
8547 return dfixed_trunc(bandwidth);
8551 * dce8_available_bandwidth - get the min available bandwidth
8553 * @wm: watermark calculation data
8555 * Calculate the min available bandwidth used for display (CIK).
8556 * Used for display watermark bandwidth calculations
8557 * Returns the min available bandwidth in MBytes/s
8559 static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
8561 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
8562 u32 dram_bandwidth = dce8_dram_bandwidth(wm);
8563 u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
8564 u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
8566 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
8570 * dce8_average_bandwidth - get the average available bandwidth
8572 * @wm: watermark calculation data
8574 * Calculate the average available bandwidth used for display (CIK).
8575 * Used for display watermark bandwidth calculations
8576 * Returns the average available bandwidth in MBytes/s
8578 static u32 dce8_average_bandwidth(struct dce8_wm_params *wm)
8580 /* Calculate the display mode Average Bandwidth
8581 * DisplayMode should contain the source and destination dimensions,
8585 fixed20_12 line_time;
8586 fixed20_12 src_width;
8587 fixed20_12 bandwidth;
8590 a.full = dfixed_const(1000);
8591 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
8592 line_time.full = dfixed_div(line_time, a);
8593 bpp.full = dfixed_const(wm->bytes_per_pixel);
8594 src_width.full = dfixed_const(wm->src_width);
8595 bandwidth.full = dfixed_mul(src_width, bpp);
8596 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
8597 bandwidth.full = dfixed_div(bandwidth, line_time);
8599 return dfixed_trunc(bandwidth);
8603 * dce8_latency_watermark - get the latency watermark
8605 * @wm: watermark calculation data
8607 * Calculate the latency watermark (CIK).
8608 * Used for display watermark bandwidth calculations
8609 * Returns the latency watermark in ns
8611 static u32 dce8_latency_watermark(struct dce8_wm_params *wm)
8613 /* First calculate the latency in ns */
8614 u32 mc_latency = 2000; /* 2000 ns. */
8615 u32 available_bandwidth = dce8_available_bandwidth(wm);
8616 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
8617 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
8618 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
8619 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
8620 (wm->num_heads * cursor_line_pair_return_time);
8621 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
8622 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
8623 u32 tmp, dmif_size = 12288;
8626 if (wm->num_heads == 0)
8629 a.full = dfixed_const(2);
8630 b.full = dfixed_const(1);
8631 if ((wm->vsc.full > a.full) ||
8632 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
8634 ((wm->vsc.full >= a.full) && wm->interlaced))
8635 max_src_lines_per_dst_line = 4;
8637 max_src_lines_per_dst_line = 2;
8639 a.full = dfixed_const(available_bandwidth);
8640 b.full = dfixed_const(wm->num_heads);
8641 a.full = dfixed_div(a, b);
8643 b.full = dfixed_const(mc_latency + 512);
8644 c.full = dfixed_const(wm->disp_clk);
8645 b.full = dfixed_div(b, c);
8647 c.full = dfixed_const(dmif_size);
8648 b.full = dfixed_div(c, b);
8650 tmp = min(dfixed_trunc(a), dfixed_trunc(b));
8652 b.full = dfixed_const(1000);
8653 c.full = dfixed_const(wm->disp_clk);
8654 b.full = dfixed_div(c, b);
8655 c.full = dfixed_const(wm->bytes_per_pixel);
8656 b.full = dfixed_mul(b, c);
8658 lb_fill_bw = min(tmp, dfixed_trunc(b));
8660 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
8661 b.full = dfixed_const(1000);
8662 c.full = dfixed_const(lb_fill_bw);
8663 b.full = dfixed_div(c, b);
8664 a.full = dfixed_div(a, b);
8665 line_fill_time = dfixed_trunc(a);
8667 if (line_fill_time < wm->active_time)
8670 return latency + (line_fill_time - wm->active_time);
8675 * dce8_average_bandwidth_vs_dram_bandwidth_for_display - check
8676 * average and available dram bandwidth
8678 * @wm: watermark calculation data
8680 * Check if the display average bandwidth fits in the display
8681 * dram bandwidth (CIK).
8682 * Used for display watermark bandwidth calculations
8683 * Returns true if the display fits, false if not.
8685 static bool dce8_average_bandwidth_vs_dram_bandwidth_for_display(struct dce8_wm_params *wm)
8687 if (dce8_average_bandwidth(wm) <=
8688 (dce8_dram_bandwidth_for_display(wm) / wm->num_heads))
8695 * dce8_average_bandwidth_vs_available_bandwidth - check
8696 * average and available bandwidth
8698 * @wm: watermark calculation data
8700 * Check if the display average bandwidth fits in the display
8701 * available bandwidth (CIK).
8702 * Used for display watermark bandwidth calculations
8703 * Returns true if the display fits, false if not.
8705 static bool dce8_average_bandwidth_vs_available_bandwidth(struct dce8_wm_params *wm)
8707 if (dce8_average_bandwidth(wm) <=
8708 (dce8_available_bandwidth(wm) / wm->num_heads))
8715 * dce8_check_latency_hiding - check latency hiding
8717 * @wm: watermark calculation data
8719 * Check latency hiding (CIK).
8720 * Used for display watermark bandwidth calculations
8721 * Returns true if the display fits, false if not.
8723 static bool dce8_check_latency_hiding(struct dce8_wm_params *wm)
8725 u32 lb_partitions = wm->lb_size / wm->src_width;
8726 u32 line_time = wm->active_time + wm->blank_time;
8727 u32 latency_tolerant_lines;
8731 a.full = dfixed_const(1);
8732 if (wm->vsc.full > a.full)
8733 latency_tolerant_lines = 1;
8735 if (lb_partitions <= (wm->vtaps + 1))
8736 latency_tolerant_lines = 1;
8738 latency_tolerant_lines = 2;
8741 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
8743 if (dce8_latency_watermark(wm) <= latency_hiding)
8750 * dce8_program_watermarks - program display watermarks
8752 * @rdev: radeon_device pointer
8753 * @radeon_crtc: the selected display controller
8754 * @lb_size: line buffer size
8755 * @num_heads: number of display controllers in use
8757 * Calculate and program the display watermarks for the
8758 * selected display controller (CIK).
8760 static void dce8_program_watermarks(struct radeon_device *rdev,
8761 struct radeon_crtc *radeon_crtc,
8762 u32 lb_size, u32 num_heads)
8764 struct drm_display_mode *mode = &radeon_crtc->base.mode;
8765 struct dce8_wm_params wm_low, wm_high;
8768 u32 latency_watermark_a = 0, latency_watermark_b = 0;
8771 if (radeon_crtc->base.enabled && num_heads && mode) {
8772 pixel_period = 1000000 / (u32)mode->clock;
8773 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
8775 /* watermark for high clocks */
8776 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8777 rdev->pm.dpm_enabled) {
8779 radeon_dpm_get_mclk(rdev, false) * 10;
8781 radeon_dpm_get_sclk(rdev, false) * 10;
8783 wm_high.yclk = rdev->pm.current_mclk * 10;
8784 wm_high.sclk = rdev->pm.current_sclk * 10;
8787 wm_high.disp_clk = mode->clock;
8788 wm_high.src_width = mode->crtc_hdisplay;
8789 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
8790 wm_high.blank_time = line_time - wm_high.active_time;
8791 wm_high.interlaced = false;
8792 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
8793 wm_high.interlaced = true;
8794 wm_high.vsc = radeon_crtc->vsc;
8796 if (radeon_crtc->rmx_type != RMX_OFF)
8798 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
8799 wm_high.lb_size = lb_size;
8800 wm_high.dram_channels = cik_get_number_of_dram_channels(rdev);
8801 wm_high.num_heads = num_heads;
8803 /* set for high clocks */
8804 latency_watermark_a = min(dce8_latency_watermark(&wm_high), (u32)65535);
8806 /* possibly force display priority to high */
8807 /* should really do this at mode validation time... */
8808 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
8809 !dce8_average_bandwidth_vs_available_bandwidth(&wm_high) ||
8810 !dce8_check_latency_hiding(&wm_high) ||
8811 (rdev->disp_priority == 2)) {
8812 DRM_DEBUG_KMS("force priority to high\n");
8815 /* watermark for low clocks */
8816 if ((rdev->pm.pm_method == PM_METHOD_DPM) &&
8817 rdev->pm.dpm_enabled) {
8819 radeon_dpm_get_mclk(rdev, true) * 10;
8821 radeon_dpm_get_sclk(rdev, true) * 10;
8823 wm_low.yclk = rdev->pm.current_mclk * 10;
8824 wm_low.sclk = rdev->pm.current_sclk * 10;
8827 wm_low.disp_clk = mode->clock;
8828 wm_low.src_width = mode->crtc_hdisplay;
8829 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
8830 wm_low.blank_time = line_time - wm_low.active_time;
8831 wm_low.interlaced = false;
8832 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
8833 wm_low.interlaced = true;
8834 wm_low.vsc = radeon_crtc->vsc;
8836 if (radeon_crtc->rmx_type != RMX_OFF)
8838 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
8839 wm_low.lb_size = lb_size;
8840 wm_low.dram_channels = cik_get_number_of_dram_channels(rdev);
8841 wm_low.num_heads = num_heads;
8843 /* set for low clocks */
8844 latency_watermark_b = min(dce8_latency_watermark(&wm_low), (u32)65535);
8846 /* possibly force display priority to high */
8847 /* should really do this at mode validation time... */
8848 if (!dce8_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
8849 !dce8_average_bandwidth_vs_available_bandwidth(&wm_low) ||
8850 !dce8_check_latency_hiding(&wm_low) ||
8851 (rdev->disp_priority == 2)) {
8852 DRM_DEBUG_KMS("force priority to high\n");
8857 wm_mask = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
8859 tmp &= ~LATENCY_WATERMARK_MASK(3);
8860 tmp |= LATENCY_WATERMARK_MASK(1);
8861 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
8862 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
8863 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
8864 LATENCY_HIGH_WATERMARK(line_time)));
8866 tmp = RREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset);
8867 tmp &= ~LATENCY_WATERMARK_MASK(3);
8868 tmp |= LATENCY_WATERMARK_MASK(2);
8869 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, tmp);
8870 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
8871 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
8872 LATENCY_HIGH_WATERMARK(line_time)));
8873 /* restore original selection */
8874 WREG32(DPG_WATERMARK_MASK_CONTROL + radeon_crtc->crtc_offset, wm_mask);
8876 /* save values for DPM */
8877 radeon_crtc->line_time = line_time;
8878 radeon_crtc->wm_high = latency_watermark_a;
8879 radeon_crtc->wm_low = latency_watermark_b;
8883 * dce8_bandwidth_update - program display watermarks
8885 * @rdev: radeon_device pointer
8887 * Calculate and program the display watermarks and line
8888 * buffer allocation (CIK).
8890 void dce8_bandwidth_update(struct radeon_device *rdev)
8892 struct drm_display_mode *mode = NULL;
8893 u32 num_heads = 0, lb_size;
8896 radeon_update_display_priority(rdev);
8898 for (i = 0; i < rdev->num_crtc; i++) {
8899 if (rdev->mode_info.crtcs[i]->base.enabled)
8902 for (i = 0; i < rdev->num_crtc; i++) {
8903 mode = &rdev->mode_info.crtcs[i]->base.mode;
8904 lb_size = dce8_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode);
8905 dce8_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
8910 * cik_get_gpu_clock_counter - return GPU clock counter snapshot
8912 * @rdev: radeon_device pointer
8914 * Fetches a GPU clock counter snapshot (SI).
8915 * Returns the 64 bit clock counter snapshot.
8917 uint64_t cik_get_gpu_clock_counter(struct radeon_device *rdev)
8921 mutex_lock(&rdev->gpu_clock_mutex);
8922 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
8923 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
8924 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
8925 mutex_unlock(&rdev->gpu_clock_mutex);
8929 static int cik_set_uvd_clock(struct radeon_device *rdev, u32 clock,
8930 u32 cntl_reg, u32 status_reg)
8933 struct atom_clock_dividers dividers;
8936 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
8937 clock, false, ÷rs);
8941 tmp = RREG32_SMC(cntl_reg);
8942 tmp &= ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK);
8943 tmp |= dividers.post_divider;
8944 WREG32_SMC(cntl_reg, tmp);
8946 for (i = 0; i < 100; i++) {
8947 if (RREG32_SMC(status_reg) & DCLK_STATUS)
8957 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
8961 r = cik_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
8965 r = cik_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
8969 static void cik_pcie_gen3_enable(struct radeon_device *rdev)
8971 struct pci_dev *root = rdev->pdev->bus->self;
8972 int bridge_pos, gpu_pos;
8973 u32 speed_cntl, mask, current_data_rate;
8977 if (radeon_pcie_gen2 == 0)
8980 if (rdev->flags & RADEON_IS_IGP)
8983 if (!(rdev->flags & RADEON_IS_PCIE))
8986 ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
8990 if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
8993 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
8994 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
8995 LC_CURRENT_DATA_RATE_SHIFT;
8996 if (mask & DRM_PCIE_SPEED_80) {
8997 if (current_data_rate == 2) {
8998 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
9001 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
9002 } else if (mask & DRM_PCIE_SPEED_50) {
9003 if (current_data_rate == 1) {
9004 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
9007 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
9010 bridge_pos = pci_pcie_cap(root);
9014 gpu_pos = pci_pcie_cap(rdev->pdev);
9018 if (mask & DRM_PCIE_SPEED_80) {
9019 /* re-try equalization if gen3 is not already enabled */
9020 if (current_data_rate != 2) {
9021 u16 bridge_cfg, gpu_cfg;
9022 u16 bridge_cfg2, gpu_cfg2;
9023 u32 max_lw, current_lw, tmp;
9025 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9026 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9028 tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
9029 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9031 tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
9032 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9034 tmp = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9035 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
9036 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
9038 if (current_lw < max_lw) {
9039 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9040 if (tmp & LC_RENEGOTIATION_SUPPORT) {
9041 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
9042 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
9043 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
9044 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
9048 for (i = 0; i < 10; i++) {
9050 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
9051 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
9054 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
9055 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
9057 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
9058 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
9060 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9061 tmp |= LC_SET_QUIESCE;
9062 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9064 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9066 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9071 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
9072 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9073 tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
9074 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
9076 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
9077 tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
9078 tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
9079 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
9082 pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
9083 tmp16 &= ~((1 << 4) | (7 << 9));
9084 tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
9085 pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
9087 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9088 tmp16 &= ~((1 << 4) | (7 << 9));
9089 tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
9090 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9092 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
9093 tmp &= ~LC_SET_QUIESCE;
9094 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
9099 /* set the link speed */
9100 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
9101 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
9102 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9104 pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
9106 if (mask & DRM_PCIE_SPEED_80)
9107 tmp16 |= 3; /* gen3 */
9108 else if (mask & DRM_PCIE_SPEED_50)
9109 tmp16 |= 2; /* gen2 */
9111 tmp16 |= 1; /* gen1 */
9112 pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
9114 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9115 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
9116 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
9118 for (i = 0; i < rdev->usec_timeout; i++) {
9119 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
9120 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
9126 static void cik_program_aspm(struct radeon_device *rdev)
9129 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
9130 bool disable_clkreq = false;
9132 if (radeon_aspm == 0)
9135 /* XXX double check IGPs */
9136 if (rdev->flags & RADEON_IS_IGP)
9139 if (!(rdev->flags & RADEON_IS_PCIE))
9142 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9143 data &= ~LC_XMIT_N_FTS_MASK;
9144 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
9146 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
9148 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
9149 data |= LC_GO_TO_RECOVERY;
9151 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
9153 orig = data = RREG32_PCIE_PORT(PCIE_P_CNTL);
9154 data |= P_IGNORE_EDB_ERR;
9156 WREG32_PCIE_PORT(PCIE_P_CNTL, data);
9158 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9159 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
9160 data |= LC_PMI_TO_L1_DIS;
9162 data |= LC_L0S_INACTIVITY(7);
9165 data |= LC_L1_INACTIVITY(7);
9166 data &= ~LC_PMI_TO_L1_DIS;
9168 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9170 if (!disable_plloff_in_l1) {
9171 bool clk_req_support;
9173 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0);
9174 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9175 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9177 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_0, data);
9179 orig = data = RREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1);
9180 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9181 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9183 WREG32_PCIE_PORT(PB0_PIF_PWRDOWN_1, data);
9185 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0);
9186 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
9187 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
9189 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_0, data);
9191 orig = data = RREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1);
9192 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
9193 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
9195 WREG32_PCIE_PORT(PB1_PIF_PWRDOWN_1, data);
9197 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
9198 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
9199 data |= LC_DYN_LANES_PWR_STATE(3);
9201 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
9203 if (!disable_clkreq) {
9204 struct pci_dev *root = rdev->pdev->bus->self;
9207 clk_req_support = false;
9208 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
9209 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
9210 clk_req_support = true;
9212 clk_req_support = false;
9215 if (clk_req_support) {
9216 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
9217 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
9219 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
9221 orig = data = RREG32_SMC(THM_CLK_CNTL);
9222 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
9223 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
9225 WREG32_SMC(THM_CLK_CNTL, data);
9227 orig = data = RREG32_SMC(MISC_CLK_CTRL);
9228 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
9229 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
9231 WREG32_SMC(MISC_CLK_CTRL, data);
9233 orig = data = RREG32_SMC(CG_CLKPIN_CNTL);
9234 data &= ~BCLK_AS_XCLK;
9236 WREG32_SMC(CG_CLKPIN_CNTL, data);
9238 orig = data = RREG32_SMC(CG_CLKPIN_CNTL_2);
9239 data &= ~FORCE_BIF_REFCLK_EN;
9241 WREG32_SMC(CG_CLKPIN_CNTL_2, data);
9243 orig = data = RREG32_SMC(MPLL_BYPASSCLK_SEL);
9244 data &= ~MPLL_CLKOUT_SEL_MASK;
9245 data |= MPLL_CLKOUT_SEL(4);
9247 WREG32_SMC(MPLL_BYPASSCLK_SEL, data);
9252 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
9255 orig = data = RREG32_PCIE_PORT(PCIE_CNTL2);
9256 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
9258 WREG32_PCIE_PORT(PCIE_CNTL2, data);
9261 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
9262 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
9263 data = RREG32_PCIE_PORT(PCIE_LC_STATUS1);
9264 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
9265 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
9266 data &= ~LC_L0S_INACTIVITY_MASK;
9268 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);