2 * Copyright 2010 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
22 * Authors: Alex Deucher
24 #include <linux/firmware.h>
25 #include <linux/platform_device.h>
26 #include <linux/slab.h>
29 #include "radeon_asic.h"
30 #include <drm/radeon_drm.h>
31 #include "evergreend.h"
34 #include "evergreen_reg.h"
35 #include "evergreen_blit_shaders.h"
36 #include "radeon_ucode.h"
38 static const u32 crtc_offsets[6] =
40 EVERGREEN_CRTC0_REGISTER_OFFSET,
41 EVERGREEN_CRTC1_REGISTER_OFFSET,
42 EVERGREEN_CRTC2_REGISTER_OFFSET,
43 EVERGREEN_CRTC3_REGISTER_OFFSET,
44 EVERGREEN_CRTC4_REGISTER_OFFSET,
45 EVERGREEN_CRTC5_REGISTER_OFFSET
48 #include "clearstate_evergreen.h"
50 static const u32 sumo_rlc_save_restore_register_list[] =
135 static void evergreen_gpu_init(struct radeon_device *rdev);
136 void evergreen_fini(struct radeon_device *rdev);
137 void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
138 void evergreen_program_aspm(struct radeon_device *rdev);
139 extern void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
140 int ring, u32 cp_int_cntl);
141 extern void cayman_vm_decode_fault(struct radeon_device *rdev,
142 u32 status, u32 addr);
143 void cik_init_cp_pg_table(struct radeon_device *rdev);
145 extern u32 si_get_csb_size(struct radeon_device *rdev);
146 extern void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
147 extern u32 cik_get_csb_size(struct radeon_device *rdev);
148 extern void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer);
149 extern void rv770_set_clk_bypass_mode(struct radeon_device *rdev);
151 static const u32 evergreen_golden_registers[] =
153 0x3f90, 0xffff0000, 0xff000000,
154 0x9148, 0xffff0000, 0xff000000,
155 0x3f94, 0xffff0000, 0xff000000,
156 0x914c, 0xffff0000, 0xff000000,
157 0x9b7c, 0xffffffff, 0x00000000,
158 0x8a14, 0xffffffff, 0x00000007,
159 0x8b10, 0xffffffff, 0x00000000,
160 0x960c, 0xffffffff, 0x54763210,
161 0x88c4, 0xffffffff, 0x000000c2,
162 0x88d4, 0xffffffff, 0x00000010,
163 0x8974, 0xffffffff, 0x00000000,
164 0xc78, 0x00000080, 0x00000080,
165 0x5eb4, 0xffffffff, 0x00000002,
166 0x5e78, 0xffffffff, 0x001000f0,
167 0x6104, 0x01000300, 0x00000000,
168 0x5bc0, 0x00300000, 0x00000000,
169 0x7030, 0xffffffff, 0x00000011,
170 0x7c30, 0xffffffff, 0x00000011,
171 0x10830, 0xffffffff, 0x00000011,
172 0x11430, 0xffffffff, 0x00000011,
173 0x12030, 0xffffffff, 0x00000011,
174 0x12c30, 0xffffffff, 0x00000011,
175 0xd02c, 0xffffffff, 0x08421000,
176 0x240c, 0xffffffff, 0x00000380,
177 0x8b24, 0xffffffff, 0x00ff0fff,
178 0x28a4c, 0x06000000, 0x06000000,
179 0x10c, 0x00000001, 0x00000001,
180 0x8d00, 0xffffffff, 0x100e4848,
181 0x8d04, 0xffffffff, 0x00164745,
182 0x8c00, 0xffffffff, 0xe4000003,
183 0x8c04, 0xffffffff, 0x40600060,
184 0x8c08, 0xffffffff, 0x001c001c,
185 0x8cf0, 0xffffffff, 0x08e00620,
186 0x8c20, 0xffffffff, 0x00800080,
187 0x8c24, 0xffffffff, 0x00800080,
188 0x8c18, 0xffffffff, 0x20202078,
189 0x8c1c, 0xffffffff, 0x00001010,
190 0x28350, 0xffffffff, 0x00000000,
191 0xa008, 0xffffffff, 0x00010000,
192 0x5c4, 0xffffffff, 0x00000001,
193 0x9508, 0xffffffff, 0x00000002,
194 0x913c, 0x0000000f, 0x0000000a
197 static const u32 evergreen_golden_registers2[] =
199 0x2f4c, 0xffffffff, 0x00000000,
200 0x54f4, 0xffffffff, 0x00000000,
201 0x54f0, 0xffffffff, 0x00000000,
202 0x5498, 0xffffffff, 0x00000000,
203 0x549c, 0xffffffff, 0x00000000,
204 0x5494, 0xffffffff, 0x00000000,
205 0x53cc, 0xffffffff, 0x00000000,
206 0x53c8, 0xffffffff, 0x00000000,
207 0x53c4, 0xffffffff, 0x00000000,
208 0x53c0, 0xffffffff, 0x00000000,
209 0x53bc, 0xffffffff, 0x00000000,
210 0x53b8, 0xffffffff, 0x00000000,
211 0x53b4, 0xffffffff, 0x00000000,
212 0x53b0, 0xffffffff, 0x00000000
215 static const u32 cypress_mgcg_init[] =
217 0x802c, 0xffffffff, 0xc0000000,
218 0x5448, 0xffffffff, 0x00000100,
219 0x55e4, 0xffffffff, 0x00000100,
220 0x160c, 0xffffffff, 0x00000100,
221 0x5644, 0xffffffff, 0x00000100,
222 0xc164, 0xffffffff, 0x00000100,
223 0x8a18, 0xffffffff, 0x00000100,
224 0x897c, 0xffffffff, 0x06000100,
225 0x8b28, 0xffffffff, 0x00000100,
226 0x9144, 0xffffffff, 0x00000100,
227 0x9a60, 0xffffffff, 0x00000100,
228 0x9868, 0xffffffff, 0x00000100,
229 0x8d58, 0xffffffff, 0x00000100,
230 0x9510, 0xffffffff, 0x00000100,
231 0x949c, 0xffffffff, 0x00000100,
232 0x9654, 0xffffffff, 0x00000100,
233 0x9030, 0xffffffff, 0x00000100,
234 0x9034, 0xffffffff, 0x00000100,
235 0x9038, 0xffffffff, 0x00000100,
236 0x903c, 0xffffffff, 0x00000100,
237 0x9040, 0xffffffff, 0x00000100,
238 0xa200, 0xffffffff, 0x00000100,
239 0xa204, 0xffffffff, 0x00000100,
240 0xa208, 0xffffffff, 0x00000100,
241 0xa20c, 0xffffffff, 0x00000100,
242 0x971c, 0xffffffff, 0x00000100,
243 0x977c, 0xffffffff, 0x00000100,
244 0x3f80, 0xffffffff, 0x00000100,
245 0xa210, 0xffffffff, 0x00000100,
246 0xa214, 0xffffffff, 0x00000100,
247 0x4d8, 0xffffffff, 0x00000100,
248 0x9784, 0xffffffff, 0x00000100,
249 0x9698, 0xffffffff, 0x00000100,
250 0x4d4, 0xffffffff, 0x00000200,
251 0x30cc, 0xffffffff, 0x00000100,
252 0xd0c0, 0xffffffff, 0xff000100,
253 0x802c, 0xffffffff, 0x40000000,
254 0x915c, 0xffffffff, 0x00010000,
255 0x9160, 0xffffffff, 0x00030002,
256 0x9178, 0xffffffff, 0x00070000,
257 0x917c, 0xffffffff, 0x00030002,
258 0x9180, 0xffffffff, 0x00050004,
259 0x918c, 0xffffffff, 0x00010006,
260 0x9190, 0xffffffff, 0x00090008,
261 0x9194, 0xffffffff, 0x00070000,
262 0x9198, 0xffffffff, 0x00030002,
263 0x919c, 0xffffffff, 0x00050004,
264 0x91a8, 0xffffffff, 0x00010006,
265 0x91ac, 0xffffffff, 0x00090008,
266 0x91b0, 0xffffffff, 0x00070000,
267 0x91b4, 0xffffffff, 0x00030002,
268 0x91b8, 0xffffffff, 0x00050004,
269 0x91c4, 0xffffffff, 0x00010006,
270 0x91c8, 0xffffffff, 0x00090008,
271 0x91cc, 0xffffffff, 0x00070000,
272 0x91d0, 0xffffffff, 0x00030002,
273 0x91d4, 0xffffffff, 0x00050004,
274 0x91e0, 0xffffffff, 0x00010006,
275 0x91e4, 0xffffffff, 0x00090008,
276 0x91e8, 0xffffffff, 0x00000000,
277 0x91ec, 0xffffffff, 0x00070000,
278 0x91f0, 0xffffffff, 0x00030002,
279 0x91f4, 0xffffffff, 0x00050004,
280 0x9200, 0xffffffff, 0x00010006,
281 0x9204, 0xffffffff, 0x00090008,
282 0x9208, 0xffffffff, 0x00070000,
283 0x920c, 0xffffffff, 0x00030002,
284 0x9210, 0xffffffff, 0x00050004,
285 0x921c, 0xffffffff, 0x00010006,
286 0x9220, 0xffffffff, 0x00090008,
287 0x9224, 0xffffffff, 0x00070000,
288 0x9228, 0xffffffff, 0x00030002,
289 0x922c, 0xffffffff, 0x00050004,
290 0x9238, 0xffffffff, 0x00010006,
291 0x923c, 0xffffffff, 0x00090008,
292 0x9240, 0xffffffff, 0x00070000,
293 0x9244, 0xffffffff, 0x00030002,
294 0x9248, 0xffffffff, 0x00050004,
295 0x9254, 0xffffffff, 0x00010006,
296 0x9258, 0xffffffff, 0x00090008,
297 0x925c, 0xffffffff, 0x00070000,
298 0x9260, 0xffffffff, 0x00030002,
299 0x9264, 0xffffffff, 0x00050004,
300 0x9270, 0xffffffff, 0x00010006,
301 0x9274, 0xffffffff, 0x00090008,
302 0x9278, 0xffffffff, 0x00070000,
303 0x927c, 0xffffffff, 0x00030002,
304 0x9280, 0xffffffff, 0x00050004,
305 0x928c, 0xffffffff, 0x00010006,
306 0x9290, 0xffffffff, 0x00090008,
307 0x9294, 0xffffffff, 0x00000000,
308 0x929c, 0xffffffff, 0x00000001,
309 0x802c, 0xffffffff, 0x40010000,
310 0x915c, 0xffffffff, 0x00010000,
311 0x9160, 0xffffffff, 0x00030002,
312 0x9178, 0xffffffff, 0x00070000,
313 0x917c, 0xffffffff, 0x00030002,
314 0x9180, 0xffffffff, 0x00050004,
315 0x918c, 0xffffffff, 0x00010006,
316 0x9190, 0xffffffff, 0x00090008,
317 0x9194, 0xffffffff, 0x00070000,
318 0x9198, 0xffffffff, 0x00030002,
319 0x919c, 0xffffffff, 0x00050004,
320 0x91a8, 0xffffffff, 0x00010006,
321 0x91ac, 0xffffffff, 0x00090008,
322 0x91b0, 0xffffffff, 0x00070000,
323 0x91b4, 0xffffffff, 0x00030002,
324 0x91b8, 0xffffffff, 0x00050004,
325 0x91c4, 0xffffffff, 0x00010006,
326 0x91c8, 0xffffffff, 0x00090008,
327 0x91cc, 0xffffffff, 0x00070000,
328 0x91d0, 0xffffffff, 0x00030002,
329 0x91d4, 0xffffffff, 0x00050004,
330 0x91e0, 0xffffffff, 0x00010006,
331 0x91e4, 0xffffffff, 0x00090008,
332 0x91e8, 0xffffffff, 0x00000000,
333 0x91ec, 0xffffffff, 0x00070000,
334 0x91f0, 0xffffffff, 0x00030002,
335 0x91f4, 0xffffffff, 0x00050004,
336 0x9200, 0xffffffff, 0x00010006,
337 0x9204, 0xffffffff, 0x00090008,
338 0x9208, 0xffffffff, 0x00070000,
339 0x920c, 0xffffffff, 0x00030002,
340 0x9210, 0xffffffff, 0x00050004,
341 0x921c, 0xffffffff, 0x00010006,
342 0x9220, 0xffffffff, 0x00090008,
343 0x9224, 0xffffffff, 0x00070000,
344 0x9228, 0xffffffff, 0x00030002,
345 0x922c, 0xffffffff, 0x00050004,
346 0x9238, 0xffffffff, 0x00010006,
347 0x923c, 0xffffffff, 0x00090008,
348 0x9240, 0xffffffff, 0x00070000,
349 0x9244, 0xffffffff, 0x00030002,
350 0x9248, 0xffffffff, 0x00050004,
351 0x9254, 0xffffffff, 0x00010006,
352 0x9258, 0xffffffff, 0x00090008,
353 0x925c, 0xffffffff, 0x00070000,
354 0x9260, 0xffffffff, 0x00030002,
355 0x9264, 0xffffffff, 0x00050004,
356 0x9270, 0xffffffff, 0x00010006,
357 0x9274, 0xffffffff, 0x00090008,
358 0x9278, 0xffffffff, 0x00070000,
359 0x927c, 0xffffffff, 0x00030002,
360 0x9280, 0xffffffff, 0x00050004,
361 0x928c, 0xffffffff, 0x00010006,
362 0x9290, 0xffffffff, 0x00090008,
363 0x9294, 0xffffffff, 0x00000000,
364 0x929c, 0xffffffff, 0x00000001,
365 0x802c, 0xffffffff, 0xc0000000
368 static const u32 redwood_mgcg_init[] =
370 0x802c, 0xffffffff, 0xc0000000,
371 0x5448, 0xffffffff, 0x00000100,
372 0x55e4, 0xffffffff, 0x00000100,
373 0x160c, 0xffffffff, 0x00000100,
374 0x5644, 0xffffffff, 0x00000100,
375 0xc164, 0xffffffff, 0x00000100,
376 0x8a18, 0xffffffff, 0x00000100,
377 0x897c, 0xffffffff, 0x06000100,
378 0x8b28, 0xffffffff, 0x00000100,
379 0x9144, 0xffffffff, 0x00000100,
380 0x9a60, 0xffffffff, 0x00000100,
381 0x9868, 0xffffffff, 0x00000100,
382 0x8d58, 0xffffffff, 0x00000100,
383 0x9510, 0xffffffff, 0x00000100,
384 0x949c, 0xffffffff, 0x00000100,
385 0x9654, 0xffffffff, 0x00000100,
386 0x9030, 0xffffffff, 0x00000100,
387 0x9034, 0xffffffff, 0x00000100,
388 0x9038, 0xffffffff, 0x00000100,
389 0x903c, 0xffffffff, 0x00000100,
390 0x9040, 0xffffffff, 0x00000100,
391 0xa200, 0xffffffff, 0x00000100,
392 0xa204, 0xffffffff, 0x00000100,
393 0xa208, 0xffffffff, 0x00000100,
394 0xa20c, 0xffffffff, 0x00000100,
395 0x971c, 0xffffffff, 0x00000100,
396 0x977c, 0xffffffff, 0x00000100,
397 0x3f80, 0xffffffff, 0x00000100,
398 0xa210, 0xffffffff, 0x00000100,
399 0xa214, 0xffffffff, 0x00000100,
400 0x4d8, 0xffffffff, 0x00000100,
401 0x9784, 0xffffffff, 0x00000100,
402 0x9698, 0xffffffff, 0x00000100,
403 0x4d4, 0xffffffff, 0x00000200,
404 0x30cc, 0xffffffff, 0x00000100,
405 0xd0c0, 0xffffffff, 0xff000100,
406 0x802c, 0xffffffff, 0x40000000,
407 0x915c, 0xffffffff, 0x00010000,
408 0x9160, 0xffffffff, 0x00030002,
409 0x9178, 0xffffffff, 0x00070000,
410 0x917c, 0xffffffff, 0x00030002,
411 0x9180, 0xffffffff, 0x00050004,
412 0x918c, 0xffffffff, 0x00010006,
413 0x9190, 0xffffffff, 0x00090008,
414 0x9194, 0xffffffff, 0x00070000,
415 0x9198, 0xffffffff, 0x00030002,
416 0x919c, 0xffffffff, 0x00050004,
417 0x91a8, 0xffffffff, 0x00010006,
418 0x91ac, 0xffffffff, 0x00090008,
419 0x91b0, 0xffffffff, 0x00070000,
420 0x91b4, 0xffffffff, 0x00030002,
421 0x91b8, 0xffffffff, 0x00050004,
422 0x91c4, 0xffffffff, 0x00010006,
423 0x91c8, 0xffffffff, 0x00090008,
424 0x91cc, 0xffffffff, 0x00070000,
425 0x91d0, 0xffffffff, 0x00030002,
426 0x91d4, 0xffffffff, 0x00050004,
427 0x91e0, 0xffffffff, 0x00010006,
428 0x91e4, 0xffffffff, 0x00090008,
429 0x91e8, 0xffffffff, 0x00000000,
430 0x91ec, 0xffffffff, 0x00070000,
431 0x91f0, 0xffffffff, 0x00030002,
432 0x91f4, 0xffffffff, 0x00050004,
433 0x9200, 0xffffffff, 0x00010006,
434 0x9204, 0xffffffff, 0x00090008,
435 0x9294, 0xffffffff, 0x00000000,
436 0x929c, 0xffffffff, 0x00000001,
437 0x802c, 0xffffffff, 0xc0000000
440 static const u32 cedar_golden_registers[] =
442 0x3f90, 0xffff0000, 0xff000000,
443 0x9148, 0xffff0000, 0xff000000,
444 0x3f94, 0xffff0000, 0xff000000,
445 0x914c, 0xffff0000, 0xff000000,
446 0x9b7c, 0xffffffff, 0x00000000,
447 0x8a14, 0xffffffff, 0x00000007,
448 0x8b10, 0xffffffff, 0x00000000,
449 0x960c, 0xffffffff, 0x54763210,
450 0x88c4, 0xffffffff, 0x000000c2,
451 0x88d4, 0xffffffff, 0x00000000,
452 0x8974, 0xffffffff, 0x00000000,
453 0xc78, 0x00000080, 0x00000080,
454 0x5eb4, 0xffffffff, 0x00000002,
455 0x5e78, 0xffffffff, 0x001000f0,
456 0x6104, 0x01000300, 0x00000000,
457 0x5bc0, 0x00300000, 0x00000000,
458 0x7030, 0xffffffff, 0x00000011,
459 0x7c30, 0xffffffff, 0x00000011,
460 0x10830, 0xffffffff, 0x00000011,
461 0x11430, 0xffffffff, 0x00000011,
462 0xd02c, 0xffffffff, 0x08421000,
463 0x240c, 0xffffffff, 0x00000380,
464 0x8b24, 0xffffffff, 0x00ff0fff,
465 0x28a4c, 0x06000000, 0x06000000,
466 0x10c, 0x00000001, 0x00000001,
467 0x8d00, 0xffffffff, 0x100e4848,
468 0x8d04, 0xffffffff, 0x00164745,
469 0x8c00, 0xffffffff, 0xe4000003,
470 0x8c04, 0xffffffff, 0x40600060,
471 0x8c08, 0xffffffff, 0x001c001c,
472 0x8cf0, 0xffffffff, 0x08e00410,
473 0x8c20, 0xffffffff, 0x00800080,
474 0x8c24, 0xffffffff, 0x00800080,
475 0x8c18, 0xffffffff, 0x20202078,
476 0x8c1c, 0xffffffff, 0x00001010,
477 0x28350, 0xffffffff, 0x00000000,
478 0xa008, 0xffffffff, 0x00010000,
479 0x5c4, 0xffffffff, 0x00000001,
480 0x9508, 0xffffffff, 0x00000002
483 static const u32 cedar_mgcg_init[] =
485 0x802c, 0xffffffff, 0xc0000000,
486 0x5448, 0xffffffff, 0x00000100,
487 0x55e4, 0xffffffff, 0x00000100,
488 0x160c, 0xffffffff, 0x00000100,
489 0x5644, 0xffffffff, 0x00000100,
490 0xc164, 0xffffffff, 0x00000100,
491 0x8a18, 0xffffffff, 0x00000100,
492 0x897c, 0xffffffff, 0x06000100,
493 0x8b28, 0xffffffff, 0x00000100,
494 0x9144, 0xffffffff, 0x00000100,
495 0x9a60, 0xffffffff, 0x00000100,
496 0x9868, 0xffffffff, 0x00000100,
497 0x8d58, 0xffffffff, 0x00000100,
498 0x9510, 0xffffffff, 0x00000100,
499 0x949c, 0xffffffff, 0x00000100,
500 0x9654, 0xffffffff, 0x00000100,
501 0x9030, 0xffffffff, 0x00000100,
502 0x9034, 0xffffffff, 0x00000100,
503 0x9038, 0xffffffff, 0x00000100,
504 0x903c, 0xffffffff, 0x00000100,
505 0x9040, 0xffffffff, 0x00000100,
506 0xa200, 0xffffffff, 0x00000100,
507 0xa204, 0xffffffff, 0x00000100,
508 0xa208, 0xffffffff, 0x00000100,
509 0xa20c, 0xffffffff, 0x00000100,
510 0x971c, 0xffffffff, 0x00000100,
511 0x977c, 0xffffffff, 0x00000100,
512 0x3f80, 0xffffffff, 0x00000100,
513 0xa210, 0xffffffff, 0x00000100,
514 0xa214, 0xffffffff, 0x00000100,
515 0x4d8, 0xffffffff, 0x00000100,
516 0x9784, 0xffffffff, 0x00000100,
517 0x9698, 0xffffffff, 0x00000100,
518 0x4d4, 0xffffffff, 0x00000200,
519 0x30cc, 0xffffffff, 0x00000100,
520 0xd0c0, 0xffffffff, 0xff000100,
521 0x802c, 0xffffffff, 0x40000000,
522 0x915c, 0xffffffff, 0x00010000,
523 0x9178, 0xffffffff, 0x00050000,
524 0x917c, 0xffffffff, 0x00030002,
525 0x918c, 0xffffffff, 0x00010004,
526 0x9190, 0xffffffff, 0x00070006,
527 0x9194, 0xffffffff, 0x00050000,
528 0x9198, 0xffffffff, 0x00030002,
529 0x91a8, 0xffffffff, 0x00010004,
530 0x91ac, 0xffffffff, 0x00070006,
531 0x91e8, 0xffffffff, 0x00000000,
532 0x9294, 0xffffffff, 0x00000000,
533 0x929c, 0xffffffff, 0x00000001,
534 0x802c, 0xffffffff, 0xc0000000
537 static const u32 juniper_mgcg_init[] =
539 0x802c, 0xffffffff, 0xc0000000,
540 0x5448, 0xffffffff, 0x00000100,
541 0x55e4, 0xffffffff, 0x00000100,
542 0x160c, 0xffffffff, 0x00000100,
543 0x5644, 0xffffffff, 0x00000100,
544 0xc164, 0xffffffff, 0x00000100,
545 0x8a18, 0xffffffff, 0x00000100,
546 0x897c, 0xffffffff, 0x06000100,
547 0x8b28, 0xffffffff, 0x00000100,
548 0x9144, 0xffffffff, 0x00000100,
549 0x9a60, 0xffffffff, 0x00000100,
550 0x9868, 0xffffffff, 0x00000100,
551 0x8d58, 0xffffffff, 0x00000100,
552 0x9510, 0xffffffff, 0x00000100,
553 0x949c, 0xffffffff, 0x00000100,
554 0x9654, 0xffffffff, 0x00000100,
555 0x9030, 0xffffffff, 0x00000100,
556 0x9034, 0xffffffff, 0x00000100,
557 0x9038, 0xffffffff, 0x00000100,
558 0x903c, 0xffffffff, 0x00000100,
559 0x9040, 0xffffffff, 0x00000100,
560 0xa200, 0xffffffff, 0x00000100,
561 0xa204, 0xffffffff, 0x00000100,
562 0xa208, 0xffffffff, 0x00000100,
563 0xa20c, 0xffffffff, 0x00000100,
564 0x971c, 0xffffffff, 0x00000100,
565 0xd0c0, 0xffffffff, 0xff000100,
566 0x802c, 0xffffffff, 0x40000000,
567 0x915c, 0xffffffff, 0x00010000,
568 0x9160, 0xffffffff, 0x00030002,
569 0x9178, 0xffffffff, 0x00070000,
570 0x917c, 0xffffffff, 0x00030002,
571 0x9180, 0xffffffff, 0x00050004,
572 0x918c, 0xffffffff, 0x00010006,
573 0x9190, 0xffffffff, 0x00090008,
574 0x9194, 0xffffffff, 0x00070000,
575 0x9198, 0xffffffff, 0x00030002,
576 0x919c, 0xffffffff, 0x00050004,
577 0x91a8, 0xffffffff, 0x00010006,
578 0x91ac, 0xffffffff, 0x00090008,
579 0x91b0, 0xffffffff, 0x00070000,
580 0x91b4, 0xffffffff, 0x00030002,
581 0x91b8, 0xffffffff, 0x00050004,
582 0x91c4, 0xffffffff, 0x00010006,
583 0x91c8, 0xffffffff, 0x00090008,
584 0x91cc, 0xffffffff, 0x00070000,
585 0x91d0, 0xffffffff, 0x00030002,
586 0x91d4, 0xffffffff, 0x00050004,
587 0x91e0, 0xffffffff, 0x00010006,
588 0x91e4, 0xffffffff, 0x00090008,
589 0x91e8, 0xffffffff, 0x00000000,
590 0x91ec, 0xffffffff, 0x00070000,
591 0x91f0, 0xffffffff, 0x00030002,
592 0x91f4, 0xffffffff, 0x00050004,
593 0x9200, 0xffffffff, 0x00010006,
594 0x9204, 0xffffffff, 0x00090008,
595 0x9208, 0xffffffff, 0x00070000,
596 0x920c, 0xffffffff, 0x00030002,
597 0x9210, 0xffffffff, 0x00050004,
598 0x921c, 0xffffffff, 0x00010006,
599 0x9220, 0xffffffff, 0x00090008,
600 0x9224, 0xffffffff, 0x00070000,
601 0x9228, 0xffffffff, 0x00030002,
602 0x922c, 0xffffffff, 0x00050004,
603 0x9238, 0xffffffff, 0x00010006,
604 0x923c, 0xffffffff, 0x00090008,
605 0x9240, 0xffffffff, 0x00070000,
606 0x9244, 0xffffffff, 0x00030002,
607 0x9248, 0xffffffff, 0x00050004,
608 0x9254, 0xffffffff, 0x00010006,
609 0x9258, 0xffffffff, 0x00090008,
610 0x925c, 0xffffffff, 0x00070000,
611 0x9260, 0xffffffff, 0x00030002,
612 0x9264, 0xffffffff, 0x00050004,
613 0x9270, 0xffffffff, 0x00010006,
614 0x9274, 0xffffffff, 0x00090008,
615 0x9278, 0xffffffff, 0x00070000,
616 0x927c, 0xffffffff, 0x00030002,
617 0x9280, 0xffffffff, 0x00050004,
618 0x928c, 0xffffffff, 0x00010006,
619 0x9290, 0xffffffff, 0x00090008,
620 0x9294, 0xffffffff, 0x00000000,
621 0x929c, 0xffffffff, 0x00000001,
622 0x802c, 0xffffffff, 0xc0000000,
623 0x977c, 0xffffffff, 0x00000100,
624 0x3f80, 0xffffffff, 0x00000100,
625 0xa210, 0xffffffff, 0x00000100,
626 0xa214, 0xffffffff, 0x00000100,
627 0x4d8, 0xffffffff, 0x00000100,
628 0x9784, 0xffffffff, 0x00000100,
629 0x9698, 0xffffffff, 0x00000100,
630 0x4d4, 0xffffffff, 0x00000200,
631 0x30cc, 0xffffffff, 0x00000100,
632 0x802c, 0xffffffff, 0xc0000000
635 static const u32 supersumo_golden_registers[] =
637 0x5eb4, 0xffffffff, 0x00000002,
638 0x5c4, 0xffffffff, 0x00000001,
639 0x7030, 0xffffffff, 0x00000011,
640 0x7c30, 0xffffffff, 0x00000011,
641 0x6104, 0x01000300, 0x00000000,
642 0x5bc0, 0x00300000, 0x00000000,
643 0x8c04, 0xffffffff, 0x40600060,
644 0x8c08, 0xffffffff, 0x001c001c,
645 0x8c20, 0xffffffff, 0x00800080,
646 0x8c24, 0xffffffff, 0x00800080,
647 0x8c18, 0xffffffff, 0x20202078,
648 0x8c1c, 0xffffffff, 0x00001010,
649 0x918c, 0xffffffff, 0x00010006,
650 0x91a8, 0xffffffff, 0x00010006,
651 0x91c4, 0xffffffff, 0x00010006,
652 0x91e0, 0xffffffff, 0x00010006,
653 0x9200, 0xffffffff, 0x00010006,
654 0x9150, 0xffffffff, 0x6e944040,
655 0x917c, 0xffffffff, 0x00030002,
656 0x9180, 0xffffffff, 0x00050004,
657 0x9198, 0xffffffff, 0x00030002,
658 0x919c, 0xffffffff, 0x00050004,
659 0x91b4, 0xffffffff, 0x00030002,
660 0x91b8, 0xffffffff, 0x00050004,
661 0x91d0, 0xffffffff, 0x00030002,
662 0x91d4, 0xffffffff, 0x00050004,
663 0x91f0, 0xffffffff, 0x00030002,
664 0x91f4, 0xffffffff, 0x00050004,
665 0x915c, 0xffffffff, 0x00010000,
666 0x9160, 0xffffffff, 0x00030002,
667 0x3f90, 0xffff0000, 0xff000000,
668 0x9178, 0xffffffff, 0x00070000,
669 0x9194, 0xffffffff, 0x00070000,
670 0x91b0, 0xffffffff, 0x00070000,
671 0x91cc, 0xffffffff, 0x00070000,
672 0x91ec, 0xffffffff, 0x00070000,
673 0x9148, 0xffff0000, 0xff000000,
674 0x9190, 0xffffffff, 0x00090008,
675 0x91ac, 0xffffffff, 0x00090008,
676 0x91c8, 0xffffffff, 0x00090008,
677 0x91e4, 0xffffffff, 0x00090008,
678 0x9204, 0xffffffff, 0x00090008,
679 0x3f94, 0xffff0000, 0xff000000,
680 0x914c, 0xffff0000, 0xff000000,
681 0x929c, 0xffffffff, 0x00000001,
682 0x8a18, 0xffffffff, 0x00000100,
683 0x8b28, 0xffffffff, 0x00000100,
684 0x9144, 0xffffffff, 0x00000100,
685 0x5644, 0xffffffff, 0x00000100,
686 0x9b7c, 0xffffffff, 0x00000000,
687 0x8030, 0xffffffff, 0x0000100a,
688 0x8a14, 0xffffffff, 0x00000007,
689 0x8b24, 0xffffffff, 0x00ff0fff,
690 0x8b10, 0xffffffff, 0x00000000,
691 0x28a4c, 0x06000000, 0x06000000,
692 0x4d8, 0xffffffff, 0x00000100,
693 0x913c, 0xffff000f, 0x0100000a,
694 0x960c, 0xffffffff, 0x54763210,
695 0x88c4, 0xffffffff, 0x000000c2,
696 0x88d4, 0xffffffff, 0x00000010,
697 0x8974, 0xffffffff, 0x00000000,
698 0xc78, 0x00000080, 0x00000080,
699 0x5e78, 0xffffffff, 0x001000f0,
700 0xd02c, 0xffffffff, 0x08421000,
701 0xa008, 0xffffffff, 0x00010000,
702 0x8d00, 0xffffffff, 0x100e4848,
703 0x8d04, 0xffffffff, 0x00164745,
704 0x8c00, 0xffffffff, 0xe4000003,
705 0x8cf0, 0x1fffffff, 0x08e00620,
706 0x28350, 0xffffffff, 0x00000000,
707 0x9508, 0xffffffff, 0x00000002
710 static const u32 sumo_golden_registers[] =
712 0x900c, 0x00ffffff, 0x0017071f,
713 0x8c18, 0xffffffff, 0x10101060,
714 0x8c1c, 0xffffffff, 0x00001010,
715 0x8c30, 0x0000000f, 0x00000005,
716 0x9688, 0x0000000f, 0x00000007
719 static const u32 wrestler_golden_registers[] =
721 0x5eb4, 0xffffffff, 0x00000002,
722 0x5c4, 0xffffffff, 0x00000001,
723 0x7030, 0xffffffff, 0x00000011,
724 0x7c30, 0xffffffff, 0x00000011,
725 0x6104, 0x01000300, 0x00000000,
726 0x5bc0, 0x00300000, 0x00000000,
727 0x918c, 0xffffffff, 0x00010006,
728 0x91a8, 0xffffffff, 0x00010006,
729 0x9150, 0xffffffff, 0x6e944040,
730 0x917c, 0xffffffff, 0x00030002,
731 0x9198, 0xffffffff, 0x00030002,
732 0x915c, 0xffffffff, 0x00010000,
733 0x3f90, 0xffff0000, 0xff000000,
734 0x9178, 0xffffffff, 0x00070000,
735 0x9194, 0xffffffff, 0x00070000,
736 0x9148, 0xffff0000, 0xff000000,
737 0x9190, 0xffffffff, 0x00090008,
738 0x91ac, 0xffffffff, 0x00090008,
739 0x3f94, 0xffff0000, 0xff000000,
740 0x914c, 0xffff0000, 0xff000000,
741 0x929c, 0xffffffff, 0x00000001,
742 0x8a18, 0xffffffff, 0x00000100,
743 0x8b28, 0xffffffff, 0x00000100,
744 0x9144, 0xffffffff, 0x00000100,
745 0x9b7c, 0xffffffff, 0x00000000,
746 0x8030, 0xffffffff, 0x0000100a,
747 0x8a14, 0xffffffff, 0x00000001,
748 0x8b24, 0xffffffff, 0x00ff0fff,
749 0x8b10, 0xffffffff, 0x00000000,
750 0x28a4c, 0x06000000, 0x06000000,
751 0x4d8, 0xffffffff, 0x00000100,
752 0x913c, 0xffff000f, 0x0100000a,
753 0x960c, 0xffffffff, 0x54763210,
754 0x88c4, 0xffffffff, 0x000000c2,
755 0x88d4, 0xffffffff, 0x00000010,
756 0x8974, 0xffffffff, 0x00000000,
757 0xc78, 0x00000080, 0x00000080,
758 0x5e78, 0xffffffff, 0x001000f0,
759 0xd02c, 0xffffffff, 0x08421000,
760 0xa008, 0xffffffff, 0x00010000,
761 0x8d00, 0xffffffff, 0x100e4848,
762 0x8d04, 0xffffffff, 0x00164745,
763 0x8c00, 0xffffffff, 0xe4000003,
764 0x8cf0, 0x1fffffff, 0x08e00410,
765 0x28350, 0xffffffff, 0x00000000,
766 0x9508, 0xffffffff, 0x00000002,
767 0x900c, 0xffffffff, 0x0017071f,
768 0x8c18, 0xffffffff, 0x10101060,
769 0x8c1c, 0xffffffff, 0x00001010
772 static const u32 barts_golden_registers[] =
774 0x5eb4, 0xffffffff, 0x00000002,
775 0x5e78, 0x8f311ff1, 0x001000f0,
776 0x3f90, 0xffff0000, 0xff000000,
777 0x9148, 0xffff0000, 0xff000000,
778 0x3f94, 0xffff0000, 0xff000000,
779 0x914c, 0xffff0000, 0xff000000,
780 0xc78, 0x00000080, 0x00000080,
781 0xbd4, 0x70073777, 0x00010001,
782 0xd02c, 0xbfffff1f, 0x08421000,
783 0xd0b8, 0x03773777, 0x02011003,
784 0x5bc0, 0x00200000, 0x50100000,
785 0x98f8, 0x33773777, 0x02011003,
786 0x98fc, 0xffffffff, 0x76543210,
787 0x7030, 0x31000311, 0x00000011,
788 0x2f48, 0x00000007, 0x02011003,
789 0x6b28, 0x00000010, 0x00000012,
790 0x7728, 0x00000010, 0x00000012,
791 0x10328, 0x00000010, 0x00000012,
792 0x10f28, 0x00000010, 0x00000012,
793 0x11b28, 0x00000010, 0x00000012,
794 0x12728, 0x00000010, 0x00000012,
795 0x240c, 0x000007ff, 0x00000380,
796 0x8a14, 0xf000001f, 0x00000007,
797 0x8b24, 0x3fff3fff, 0x00ff0fff,
798 0x8b10, 0x0000ff0f, 0x00000000,
799 0x28a4c, 0x07ffffff, 0x06000000,
800 0x10c, 0x00000001, 0x00010003,
801 0xa02c, 0xffffffff, 0x0000009b,
802 0x913c, 0x0000000f, 0x0100000a,
803 0x8d00, 0xffff7f7f, 0x100e4848,
804 0x8d04, 0x00ffffff, 0x00164745,
805 0x8c00, 0xfffc0003, 0xe4000003,
806 0x8c04, 0xf8ff00ff, 0x40600060,
807 0x8c08, 0x00ff00ff, 0x001c001c,
808 0x8cf0, 0x1fff1fff, 0x08e00620,
809 0x8c20, 0x0fff0fff, 0x00800080,
810 0x8c24, 0x0fff0fff, 0x00800080,
811 0x8c18, 0xffffffff, 0x20202078,
812 0x8c1c, 0x0000ffff, 0x00001010,
813 0x28350, 0x00000f01, 0x00000000,
814 0x9508, 0x3700001f, 0x00000002,
815 0x960c, 0xffffffff, 0x54763210,
816 0x88c4, 0x001f3ae3, 0x000000c2,
817 0x88d4, 0x0000001f, 0x00000010,
818 0x8974, 0xffffffff, 0x00000000
821 static const u32 turks_golden_registers[] =
823 0x5eb4, 0xffffffff, 0x00000002,
824 0x5e78, 0x8f311ff1, 0x001000f0,
825 0x8c8, 0x00003000, 0x00001070,
826 0x8cc, 0x000fffff, 0x00040035,
827 0x3f90, 0xffff0000, 0xfff00000,
828 0x9148, 0xffff0000, 0xfff00000,
829 0x3f94, 0xffff0000, 0xfff00000,
830 0x914c, 0xffff0000, 0xfff00000,
831 0xc78, 0x00000080, 0x00000080,
832 0xbd4, 0x00073007, 0x00010002,
833 0xd02c, 0xbfffff1f, 0x08421000,
834 0xd0b8, 0x03773777, 0x02010002,
835 0x5bc0, 0x00200000, 0x50100000,
836 0x98f8, 0x33773777, 0x00010002,
837 0x98fc, 0xffffffff, 0x33221100,
838 0x7030, 0x31000311, 0x00000011,
839 0x2f48, 0x33773777, 0x00010002,
840 0x6b28, 0x00000010, 0x00000012,
841 0x7728, 0x00000010, 0x00000012,
842 0x10328, 0x00000010, 0x00000012,
843 0x10f28, 0x00000010, 0x00000012,
844 0x11b28, 0x00000010, 0x00000012,
845 0x12728, 0x00000010, 0x00000012,
846 0x240c, 0x000007ff, 0x00000380,
847 0x8a14, 0xf000001f, 0x00000007,
848 0x8b24, 0x3fff3fff, 0x00ff0fff,
849 0x8b10, 0x0000ff0f, 0x00000000,
850 0x28a4c, 0x07ffffff, 0x06000000,
851 0x10c, 0x00000001, 0x00010003,
852 0xa02c, 0xffffffff, 0x0000009b,
853 0x913c, 0x0000000f, 0x0100000a,
854 0x8d00, 0xffff7f7f, 0x100e4848,
855 0x8d04, 0x00ffffff, 0x00164745,
856 0x8c00, 0xfffc0003, 0xe4000003,
857 0x8c04, 0xf8ff00ff, 0x40600060,
858 0x8c08, 0x00ff00ff, 0x001c001c,
859 0x8cf0, 0x1fff1fff, 0x08e00410,
860 0x8c20, 0x0fff0fff, 0x00800080,
861 0x8c24, 0x0fff0fff, 0x00800080,
862 0x8c18, 0xffffffff, 0x20202078,
863 0x8c1c, 0x0000ffff, 0x00001010,
864 0x28350, 0x00000f01, 0x00000000,
865 0x9508, 0x3700001f, 0x00000002,
866 0x960c, 0xffffffff, 0x54763210,
867 0x88c4, 0x001f3ae3, 0x000000c2,
868 0x88d4, 0x0000001f, 0x00000010,
869 0x8974, 0xffffffff, 0x00000000
872 static const u32 caicos_golden_registers[] =
874 0x5eb4, 0xffffffff, 0x00000002,
875 0x5e78, 0x8f311ff1, 0x001000f0,
876 0x8c8, 0x00003420, 0x00001450,
877 0x8cc, 0x000fffff, 0x00040035,
878 0x3f90, 0xffff0000, 0xfffc0000,
879 0x9148, 0xffff0000, 0xfffc0000,
880 0x3f94, 0xffff0000, 0xfffc0000,
881 0x914c, 0xffff0000, 0xfffc0000,
882 0xc78, 0x00000080, 0x00000080,
883 0xbd4, 0x00073007, 0x00010001,
884 0xd02c, 0xbfffff1f, 0x08421000,
885 0xd0b8, 0x03773777, 0x02010001,
886 0x5bc0, 0x00200000, 0x50100000,
887 0x98f8, 0x33773777, 0x02010001,
888 0x98fc, 0xffffffff, 0x33221100,
889 0x7030, 0x31000311, 0x00000011,
890 0x2f48, 0x33773777, 0x02010001,
891 0x6b28, 0x00000010, 0x00000012,
892 0x7728, 0x00000010, 0x00000012,
893 0x10328, 0x00000010, 0x00000012,
894 0x10f28, 0x00000010, 0x00000012,
895 0x11b28, 0x00000010, 0x00000012,
896 0x12728, 0x00000010, 0x00000012,
897 0x240c, 0x000007ff, 0x00000380,
898 0x8a14, 0xf000001f, 0x00000001,
899 0x8b24, 0x3fff3fff, 0x00ff0fff,
900 0x8b10, 0x0000ff0f, 0x00000000,
901 0x28a4c, 0x07ffffff, 0x06000000,
902 0x10c, 0x00000001, 0x00010003,
903 0xa02c, 0xffffffff, 0x0000009b,
904 0x913c, 0x0000000f, 0x0100000a,
905 0x8d00, 0xffff7f7f, 0x100e4848,
906 0x8d04, 0x00ffffff, 0x00164745,
907 0x8c00, 0xfffc0003, 0xe4000003,
908 0x8c04, 0xf8ff00ff, 0x40600060,
909 0x8c08, 0x00ff00ff, 0x001c001c,
910 0x8cf0, 0x1fff1fff, 0x08e00410,
911 0x8c20, 0x0fff0fff, 0x00800080,
912 0x8c24, 0x0fff0fff, 0x00800080,
913 0x8c18, 0xffffffff, 0x20202078,
914 0x8c1c, 0x0000ffff, 0x00001010,
915 0x28350, 0x00000f01, 0x00000000,
916 0x9508, 0x3700001f, 0x00000002,
917 0x960c, 0xffffffff, 0x54763210,
918 0x88c4, 0x001f3ae3, 0x000000c2,
919 0x88d4, 0x0000001f, 0x00000010,
920 0x8974, 0xffffffff, 0x00000000
923 static void evergreen_init_golden_registers(struct radeon_device *rdev)
925 switch (rdev->family) {
928 radeon_program_register_sequence(rdev,
929 evergreen_golden_registers,
930 (const u32)ARRAY_SIZE(evergreen_golden_registers));
931 radeon_program_register_sequence(rdev,
932 evergreen_golden_registers2,
933 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
934 radeon_program_register_sequence(rdev,
936 (const u32)ARRAY_SIZE(cypress_mgcg_init));
939 radeon_program_register_sequence(rdev,
940 evergreen_golden_registers,
941 (const u32)ARRAY_SIZE(evergreen_golden_registers));
942 radeon_program_register_sequence(rdev,
943 evergreen_golden_registers2,
944 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
945 radeon_program_register_sequence(rdev,
947 (const u32)ARRAY_SIZE(juniper_mgcg_init));
950 radeon_program_register_sequence(rdev,
951 evergreen_golden_registers,
952 (const u32)ARRAY_SIZE(evergreen_golden_registers));
953 radeon_program_register_sequence(rdev,
954 evergreen_golden_registers2,
955 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
956 radeon_program_register_sequence(rdev,
958 (const u32)ARRAY_SIZE(redwood_mgcg_init));
961 radeon_program_register_sequence(rdev,
962 cedar_golden_registers,
963 (const u32)ARRAY_SIZE(cedar_golden_registers));
964 radeon_program_register_sequence(rdev,
965 evergreen_golden_registers2,
966 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
967 radeon_program_register_sequence(rdev,
969 (const u32)ARRAY_SIZE(cedar_mgcg_init));
972 radeon_program_register_sequence(rdev,
973 wrestler_golden_registers,
974 (const u32)ARRAY_SIZE(wrestler_golden_registers));
977 radeon_program_register_sequence(rdev,
978 supersumo_golden_registers,
979 (const u32)ARRAY_SIZE(supersumo_golden_registers));
982 radeon_program_register_sequence(rdev,
983 supersumo_golden_registers,
984 (const u32)ARRAY_SIZE(supersumo_golden_registers));
985 radeon_program_register_sequence(rdev,
986 sumo_golden_registers,
987 (const u32)ARRAY_SIZE(sumo_golden_registers));
990 radeon_program_register_sequence(rdev,
991 barts_golden_registers,
992 (const u32)ARRAY_SIZE(barts_golden_registers));
995 radeon_program_register_sequence(rdev,
996 turks_golden_registers,
997 (const u32)ARRAY_SIZE(turks_golden_registers));
1000 radeon_program_register_sequence(rdev,
1001 caicos_golden_registers,
1002 (const u32)ARRAY_SIZE(caicos_golden_registers));
1009 void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1010 unsigned *bankh, unsigned *mtaspect,
1011 unsigned *tile_split)
1013 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1014 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1015 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1016 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1019 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1020 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1021 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1022 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1026 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1027 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1028 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1029 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1031 switch (*mtaspect) {
1033 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1034 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1035 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1036 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1040 static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1041 u32 cntl_reg, u32 status_reg)
1044 struct atom_clock_dividers dividers;
1046 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1047 clock, false, ÷rs);
1051 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1053 for (i = 0; i < 100; i++) {
1054 if (RREG32(status_reg) & DCLK_STATUS)
1064 int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1067 u32 cg_scratch = RREG32(CG_SCRATCH1);
1069 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1072 cg_scratch &= 0xffff0000;
1073 cg_scratch |= vclk / 100; /* Mhz */
1075 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1078 cg_scratch &= 0x0000ffff;
1079 cg_scratch |= (dclk / 100) << 16; /* Mhz */
1082 WREG32(CG_SCRATCH1, cg_scratch);
1087 int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1089 /* start off with something large */
1090 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1093 /* bypass vclk and dclk with bclk */
1094 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1095 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1096 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1098 /* put PLL in bypass mode */
1099 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1101 if (!vclk || !dclk) {
1102 /* keep the Bypass mode, put PLL to sleep */
1103 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1107 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1108 16384, 0x03FFFFFF, 0, 128, 5,
1109 &fb_div, &vclk_div, &dclk_div);
1113 /* set VCO_MODE to 1 */
1114 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1116 /* toggle UPLL_SLEEP to 1 then back to 0 */
1117 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1118 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1120 /* deassert UPLL_RESET */
1121 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1125 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1129 /* assert UPLL_RESET again */
1130 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1132 /* disable spread spectrum. */
1133 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1135 /* set feedback divider */
1136 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1138 /* set ref divider to 0 */
1139 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1141 if (fb_div < 307200)
1142 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1144 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1146 /* set PDIV_A and PDIV_B */
1147 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1148 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1149 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1151 /* give the PLL some time to settle */
1154 /* deassert PLL_RESET */
1155 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1159 /* switch from bypass mode to normal mode */
1160 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1162 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1166 /* switch VCLK and DCLK selection */
1167 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1168 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1169 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1176 void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1181 readrq = pcie_get_readrq(rdev->pdev);
1182 v = ffs(readrq) - 8;
1183 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1184 * to avoid hangs or perfomance issues
1186 if ((v == 0) || (v == 6) || (v == 7))
1187 pcie_set_readrq(rdev->pdev, 512);
1190 void dce4_program_fmt(struct drm_encoder *encoder)
1192 struct drm_device *dev = encoder->dev;
1193 struct radeon_device *rdev = dev->dev_private;
1194 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1195 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1196 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1199 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
1202 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1203 bpc = radeon_get_monitor_bpc(connector);
1204 dither = radeon_connector->dither;
1207 /* LVDS/eDP FMT is set up by atom */
1208 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1211 /* not needed for analog */
1212 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1213 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1221 if (dither == RADEON_FMT_DITHER_ENABLE)
1222 /* XXX sort out optimal dither settings */
1223 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1224 FMT_SPATIAL_DITHER_EN);
1226 tmp |= FMT_TRUNCATE_EN;
1229 if (dither == RADEON_FMT_DITHER_ENABLE)
1230 /* XXX sort out optimal dither settings */
1231 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1232 FMT_RGB_RANDOM_ENABLE |
1233 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1235 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1243 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1246 static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1248 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1254 static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1258 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1259 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1268 * dce4_wait_for_vblank - vblank wait asic callback.
1270 * @rdev: radeon_device pointer
1271 * @crtc: crtc to wait for vblank on
1273 * Wait for vblank on the requested crtc (evergreen+).
1275 void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1279 if (crtc >= rdev->num_crtc)
1282 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1285 /* depending on when we hit vblank, we may be close to active; if so,
1286 * wait for another frame.
1288 while (dce4_is_in_vblank(rdev, crtc)) {
1289 if (i++ % 100 == 0) {
1290 if (!dce4_is_counter_moving(rdev, crtc))
1295 while (!dce4_is_in_vblank(rdev, crtc)) {
1296 if (i++ % 100 == 0) {
1297 if (!dce4_is_counter_moving(rdev, crtc))
1304 * radeon_irq_kms_pflip_irq_get - pre-pageflip callback.
1306 * @rdev: radeon_device pointer
1307 * @crtc: crtc to prepare for pageflip on
1309 * Pre-pageflip callback (evergreen+).
1310 * Enables the pageflip irq (vblank irq).
1312 void evergreen_pre_page_flip(struct radeon_device *rdev, int crtc)
1314 /* enable the pflip int */
1315 radeon_irq_kms_pflip_irq_get(rdev, crtc);
1319 * evergreen_post_page_flip - pos-pageflip callback.
1321 * @rdev: radeon_device pointer
1322 * @crtc: crtc to cleanup pageflip on
1324 * Post-pageflip callback (evergreen+).
1325 * Disables the pageflip irq (vblank irq).
1327 void evergreen_post_page_flip(struct radeon_device *rdev, int crtc)
1329 /* disable the pflip int */
1330 radeon_irq_kms_pflip_irq_put(rdev, crtc);
1334 * evergreen_page_flip - pageflip callback.
1336 * @rdev: radeon_device pointer
1337 * @crtc_id: crtc to cleanup pageflip on
1338 * @crtc_base: new address of the crtc (GPU MC address)
1340 * Does the actual pageflip (evergreen+).
1341 * During vblank we take the crtc lock and wait for the update_pending
1342 * bit to go high, when it does, we release the lock, and allow the
1343 * double buffered update to take place.
1344 * Returns the current update pending status.
1346 u32 evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
1348 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1349 u32 tmp = RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset);
1352 /* Lock the graphics update lock */
1353 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
1354 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1356 /* update the scanout addresses */
1357 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1358 upper_32_bits(crtc_base));
1359 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1362 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1363 upper_32_bits(crtc_base));
1364 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1367 /* Wait for update_pending to go high. */
1368 for (i = 0; i < rdev->usec_timeout; i++) {
1369 if (RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING)
1373 DRM_DEBUG("Update pending now high. Unlocking vupdate_lock.\n");
1375 /* Unlock the lock, so double-buffering can take place inside vblank */
1376 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
1377 WREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset, tmp);
1379 /* Return current update_pending status: */
1380 return RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING;
1383 /* get temperature in millidegrees */
1384 int evergreen_get_temp(struct radeon_device *rdev)
1387 int actual_temp = 0;
1389 if (rdev->family == CHIP_JUNIPER) {
1390 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1392 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1395 if (toffset & 0x100)
1396 actual_temp = temp / 2 - (0x200 - toffset);
1398 actual_temp = temp / 2 + toffset;
1400 actual_temp = actual_temp * 1000;
1403 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1408 else if (temp & 0x200)
1410 else if (temp & 0x100) {
1411 actual_temp = temp & 0x1ff;
1412 actual_temp |= ~0x1ff;
1414 actual_temp = temp & 0xff;
1416 actual_temp = (actual_temp * 1000) / 2;
1422 int sumo_get_temp(struct radeon_device *rdev)
1424 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
1425 int actual_temp = temp - 49;
1427 return actual_temp * 1000;
1431 * sumo_pm_init_profile - Initialize power profiles callback.
1433 * @rdev: radeon_device pointer
1435 * Initialize the power states used in profile mode
1436 * (sumo, trinity, SI).
1437 * Used for profile mode only.
1439 void sumo_pm_init_profile(struct radeon_device *rdev)
1444 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1445 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1446 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1447 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1450 if (rdev->flags & RADEON_IS_MOBILITY)
1451 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1453 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1455 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1456 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1457 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1458 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1460 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1461 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1462 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1463 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1465 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1466 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1467 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1468 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1470 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1471 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1472 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1473 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1476 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1477 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1478 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1479 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1480 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1481 rdev->pm.power_state[idx].num_clock_modes - 1;
1483 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1484 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1485 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1486 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1487 rdev->pm.power_state[idx].num_clock_modes - 1;
1491 * btc_pm_init_profile - Initialize power profiles callback.
1493 * @rdev: radeon_device pointer
1495 * Initialize the power states used in profile mode
1497 * Used for profile mode only.
1499 void btc_pm_init_profile(struct radeon_device *rdev)
1504 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1505 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1506 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1507 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1508 /* starting with BTC, there is one state that is used for both
1509 * MH and SH. Difference is that we always use the high clock index for
1512 if (rdev->flags & RADEON_IS_MOBILITY)
1513 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1515 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1517 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1518 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1519 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1520 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1522 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1523 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1524 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1525 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1527 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1528 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1529 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1530 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1532 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1533 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1534 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1535 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1537 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1538 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1539 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1540 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1542 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1543 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1544 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1545 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1549 * evergreen_pm_misc - set additional pm hw parameters callback.
1551 * @rdev: radeon_device pointer
1553 * Set non-clock parameters associated with a power state
1554 * (voltage, etc.) (evergreen+).
1556 void evergreen_pm_misc(struct radeon_device *rdev)
1558 int req_ps_idx = rdev->pm.requested_power_state_index;
1559 int req_cm_idx = rdev->pm.requested_clock_mode_index;
1560 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1561 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
1563 if (voltage->type == VOLTAGE_SW) {
1564 /* 0xff0x are flags rather then an actual voltage */
1565 if ((voltage->voltage & 0xff00) == 0xff00)
1567 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
1568 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1569 rdev->pm.current_vddc = voltage->voltage;
1570 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1573 /* starting with BTC, there is one state that is used for both
1574 * MH and SH. Difference is that we always use the high clock index for
1577 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1578 (rdev->family >= CHIP_BARTS) &&
1579 rdev->pm.active_crtc_count &&
1580 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1581 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1582 voltage = &rdev->pm.power_state[req_ps_idx].
1583 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1585 /* 0xff0x are flags rather then an actual voltage */
1586 if ((voltage->vddci & 0xff00) == 0xff00)
1588 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1589 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1590 rdev->pm.current_vddci = voltage->vddci;
1591 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
1597 * evergreen_pm_prepare - pre-power state change callback.
1599 * @rdev: radeon_device pointer
1601 * Prepare for a power state change (evergreen+).
1603 void evergreen_pm_prepare(struct radeon_device *rdev)
1605 struct drm_device *ddev = rdev->ddev;
1606 struct drm_crtc *crtc;
1607 struct radeon_crtc *radeon_crtc;
1610 /* disable any active CRTCs */
1611 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1612 radeon_crtc = to_radeon_crtc(crtc);
1613 if (radeon_crtc->enabled) {
1614 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1615 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1616 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1622 * evergreen_pm_finish - post-power state change callback.
1624 * @rdev: radeon_device pointer
1626 * Clean up after a power state change (evergreen+).
1628 void evergreen_pm_finish(struct radeon_device *rdev)
1630 struct drm_device *ddev = rdev->ddev;
1631 struct drm_crtc *crtc;
1632 struct radeon_crtc *radeon_crtc;
1635 /* enable any active CRTCs */
1636 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1637 radeon_crtc = to_radeon_crtc(crtc);
1638 if (radeon_crtc->enabled) {
1639 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1640 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1641 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1647 * evergreen_hpd_sense - hpd sense callback.
1649 * @rdev: radeon_device pointer
1650 * @hpd: hpd (hotplug detect) pin
1652 * Checks if a digital monitor is connected (evergreen+).
1653 * Returns true if connected, false if not connected.
1655 bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1657 bool connected = false;
1661 if (RREG32(DC_HPD1_INT_STATUS) & DC_HPDx_SENSE)
1665 if (RREG32(DC_HPD2_INT_STATUS) & DC_HPDx_SENSE)
1669 if (RREG32(DC_HPD3_INT_STATUS) & DC_HPDx_SENSE)
1673 if (RREG32(DC_HPD4_INT_STATUS) & DC_HPDx_SENSE)
1677 if (RREG32(DC_HPD5_INT_STATUS) & DC_HPDx_SENSE)
1681 if (RREG32(DC_HPD6_INT_STATUS) & DC_HPDx_SENSE)
1692 * evergreen_hpd_set_polarity - hpd set polarity callback.
1694 * @rdev: radeon_device pointer
1695 * @hpd: hpd (hotplug detect) pin
1697 * Set the polarity of the hpd pin (evergreen+).
1699 void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1700 enum radeon_hpd_id hpd)
1703 bool connected = evergreen_hpd_sense(rdev, hpd);
1707 tmp = RREG32(DC_HPD1_INT_CONTROL);
1709 tmp &= ~DC_HPDx_INT_POLARITY;
1711 tmp |= DC_HPDx_INT_POLARITY;
1712 WREG32(DC_HPD1_INT_CONTROL, tmp);
1715 tmp = RREG32(DC_HPD2_INT_CONTROL);
1717 tmp &= ~DC_HPDx_INT_POLARITY;
1719 tmp |= DC_HPDx_INT_POLARITY;
1720 WREG32(DC_HPD2_INT_CONTROL, tmp);
1723 tmp = RREG32(DC_HPD3_INT_CONTROL);
1725 tmp &= ~DC_HPDx_INT_POLARITY;
1727 tmp |= DC_HPDx_INT_POLARITY;
1728 WREG32(DC_HPD3_INT_CONTROL, tmp);
1731 tmp = RREG32(DC_HPD4_INT_CONTROL);
1733 tmp &= ~DC_HPDx_INT_POLARITY;
1735 tmp |= DC_HPDx_INT_POLARITY;
1736 WREG32(DC_HPD4_INT_CONTROL, tmp);
1739 tmp = RREG32(DC_HPD5_INT_CONTROL);
1741 tmp &= ~DC_HPDx_INT_POLARITY;
1743 tmp |= DC_HPDx_INT_POLARITY;
1744 WREG32(DC_HPD5_INT_CONTROL, tmp);
1747 tmp = RREG32(DC_HPD6_INT_CONTROL);
1749 tmp &= ~DC_HPDx_INT_POLARITY;
1751 tmp |= DC_HPDx_INT_POLARITY;
1752 WREG32(DC_HPD6_INT_CONTROL, tmp);
1760 * evergreen_hpd_init - hpd setup callback.
1762 * @rdev: radeon_device pointer
1764 * Setup the hpd pins used by the card (evergreen+).
1765 * Enable the pin, set the polarity, and enable the hpd interrupts.
1767 void evergreen_hpd_init(struct radeon_device *rdev)
1769 struct drm_device *dev = rdev->ddev;
1770 struct drm_connector *connector;
1771 unsigned enabled = 0;
1772 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1773 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
1775 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1776 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1778 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1779 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1780 /* don't try to enable hpd on eDP or LVDS avoid breaking the
1781 * aux dp channel on imac and help (but not completely fix)
1782 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1783 * also avoid interrupt storms during dpms.
1787 switch (radeon_connector->hpd.hpd) {
1789 WREG32(DC_HPD1_CONTROL, tmp);
1792 WREG32(DC_HPD2_CONTROL, tmp);
1795 WREG32(DC_HPD3_CONTROL, tmp);
1798 WREG32(DC_HPD4_CONTROL, tmp);
1801 WREG32(DC_HPD5_CONTROL, tmp);
1804 WREG32(DC_HPD6_CONTROL, tmp);
1809 radeon_hpd_set_polarity(rdev, radeon_connector->hpd.hpd);
1810 enabled |= 1 << radeon_connector->hpd.hpd;
1812 radeon_irq_kms_enable_hpd(rdev, enabled);
1816 * evergreen_hpd_fini - hpd tear down callback.
1818 * @rdev: radeon_device pointer
1820 * Tear down the hpd pins used by the card (evergreen+).
1821 * Disable the hpd interrupts.
1823 void evergreen_hpd_fini(struct radeon_device *rdev)
1825 struct drm_device *dev = rdev->ddev;
1826 struct drm_connector *connector;
1827 unsigned disabled = 0;
1829 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1830 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1831 switch (radeon_connector->hpd.hpd) {
1833 WREG32(DC_HPD1_CONTROL, 0);
1836 WREG32(DC_HPD2_CONTROL, 0);
1839 WREG32(DC_HPD3_CONTROL, 0);
1842 WREG32(DC_HPD4_CONTROL, 0);
1845 WREG32(DC_HPD5_CONTROL, 0);
1848 WREG32(DC_HPD6_CONTROL, 0);
1853 disabled |= 1 << radeon_connector->hpd.hpd;
1855 radeon_irq_kms_disable_hpd(rdev, disabled);
1858 /* watermark setup */
1860 static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1861 struct radeon_crtc *radeon_crtc,
1862 struct drm_display_mode *mode,
1863 struct drm_display_mode *other_mode)
1865 u32 tmp, buffer_alloc, i;
1866 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1869 * There are 3 line buffers, each one shared by 2 display controllers.
1870 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1871 * the display controllers. The paritioning is done via one of four
1872 * preset allocations specified in bits 2:0:
1873 * first display controller
1874 * 0 - first half of lb (3840 * 2)
1875 * 1 - first 3/4 of lb (5760 * 2)
1876 * 2 - whole lb (7680 * 2), other crtc must be disabled
1877 * 3 - first 1/4 of lb (1920 * 2)
1878 * second display controller
1879 * 4 - second half of lb (3840 * 2)
1880 * 5 - second 3/4 of lb (5760 * 2)
1881 * 6 - whole lb (7680 * 2), other crtc must be disabled
1882 * 7 - last 1/4 of lb (1920 * 2)
1884 /* this can get tricky if we have two large displays on a paired group
1885 * of crtcs. Ideally for multiple large displays we'd assign them to
1886 * non-linked crtcs for maximum line buffer allocation.
1888 if (radeon_crtc->base.enabled && mode) {
1893 tmp = 2; /* whole */
1901 /* second controller of the pair uses second half of the lb */
1902 if (radeon_crtc->crtc_id % 2)
1904 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1906 if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1907 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1908 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1909 for (i = 0; i < rdev->usec_timeout; i++) {
1910 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1911 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1917 if (radeon_crtc->base.enabled && mode) {
1922 if (ASIC_IS_DCE5(rdev))
1928 if (ASIC_IS_DCE5(rdev))
1934 if (ASIC_IS_DCE5(rdev))
1940 if (ASIC_IS_DCE5(rdev))
1947 /* controller not enabled, so no lb used */
1951 u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
1953 u32 tmp = RREG32(MC_SHARED_CHMAP);
1955 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1968 struct evergreen_wm_params {
1969 u32 dram_channels; /* number of dram channels */
1970 u32 yclk; /* bandwidth per dram data pin in kHz */
1971 u32 sclk; /* engine clock in kHz */
1972 u32 disp_clk; /* display clock in kHz */
1973 u32 src_width; /* viewport width */
1974 u32 active_time; /* active display time in ns */
1975 u32 blank_time; /* blank time in ns */
1976 bool interlaced; /* mode is interlaced */
1977 fixed20_12 vsc; /* vertical scale ratio */
1978 u32 num_heads; /* number of active crtcs */
1979 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1980 u32 lb_size; /* line buffer allocated to pipe */
1981 u32 vtaps; /* vertical scaler taps */
1984 static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1986 /* Calculate DRAM Bandwidth and the part allocated to display. */
1987 fixed20_12 dram_efficiency; /* 0.7 */
1988 fixed20_12 yclk, dram_channels, bandwidth;
1991 a.full = dfixed_const(1000);
1992 yclk.full = dfixed_const(wm->yclk);
1993 yclk.full = dfixed_div(yclk, a);
1994 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1995 a.full = dfixed_const(10);
1996 dram_efficiency.full = dfixed_const(7);
1997 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1998 bandwidth.full = dfixed_mul(dram_channels, yclk);
1999 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
2001 return dfixed_trunc(bandwidth);
2004 static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2006 /* Calculate DRAM Bandwidth and the part allocated to display. */
2007 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
2008 fixed20_12 yclk, dram_channels, bandwidth;
2011 a.full = dfixed_const(1000);
2012 yclk.full = dfixed_const(wm->yclk);
2013 yclk.full = dfixed_div(yclk, a);
2014 dram_channels.full = dfixed_const(wm->dram_channels * 4);
2015 a.full = dfixed_const(10);
2016 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
2017 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
2018 bandwidth.full = dfixed_mul(dram_channels, yclk);
2019 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
2021 return dfixed_trunc(bandwidth);
2024 static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
2026 /* Calculate the display Data return Bandwidth */
2027 fixed20_12 return_efficiency; /* 0.8 */
2028 fixed20_12 sclk, bandwidth;
2031 a.full = dfixed_const(1000);
2032 sclk.full = dfixed_const(wm->sclk);
2033 sclk.full = dfixed_div(sclk, a);
2034 a.full = dfixed_const(10);
2035 return_efficiency.full = dfixed_const(8);
2036 return_efficiency.full = dfixed_div(return_efficiency, a);
2037 a.full = dfixed_const(32);
2038 bandwidth.full = dfixed_mul(a, sclk);
2039 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2041 return dfixed_trunc(bandwidth);
2044 static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
2046 /* Calculate the DMIF Request Bandwidth */
2047 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2048 fixed20_12 disp_clk, bandwidth;
2051 a.full = dfixed_const(1000);
2052 disp_clk.full = dfixed_const(wm->disp_clk);
2053 disp_clk.full = dfixed_div(disp_clk, a);
2054 a.full = dfixed_const(10);
2055 disp_clk_request_efficiency.full = dfixed_const(8);
2056 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2057 a.full = dfixed_const(32);
2058 bandwidth.full = dfixed_mul(a, disp_clk);
2059 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2061 return dfixed_trunc(bandwidth);
2064 static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2066 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2067 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2068 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2069 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2071 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2074 static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2076 /* Calculate the display mode Average Bandwidth
2077 * DisplayMode should contain the source and destination dimensions,
2081 fixed20_12 line_time;
2082 fixed20_12 src_width;
2083 fixed20_12 bandwidth;
2086 a.full = dfixed_const(1000);
2087 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2088 line_time.full = dfixed_div(line_time, a);
2089 bpp.full = dfixed_const(wm->bytes_per_pixel);
2090 src_width.full = dfixed_const(wm->src_width);
2091 bandwidth.full = dfixed_mul(src_width, bpp);
2092 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2093 bandwidth.full = dfixed_div(bandwidth, line_time);
2095 return dfixed_trunc(bandwidth);
2098 static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2100 /* First calcualte the latency in ns */
2101 u32 mc_latency = 2000; /* 2000 ns. */
2102 u32 available_bandwidth = evergreen_available_bandwidth(wm);
2103 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2104 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2105 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2106 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2107 (wm->num_heads * cursor_line_pair_return_time);
2108 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2109 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2112 if (wm->num_heads == 0)
2115 a.full = dfixed_const(2);
2116 b.full = dfixed_const(1);
2117 if ((wm->vsc.full > a.full) ||
2118 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2120 ((wm->vsc.full >= a.full) && wm->interlaced))
2121 max_src_lines_per_dst_line = 4;
2123 max_src_lines_per_dst_line = 2;
2125 a.full = dfixed_const(available_bandwidth);
2126 b.full = dfixed_const(wm->num_heads);
2127 a.full = dfixed_div(a, b);
2129 b.full = dfixed_const(1000);
2130 c.full = dfixed_const(wm->disp_clk);
2131 b.full = dfixed_div(c, b);
2132 c.full = dfixed_const(wm->bytes_per_pixel);
2133 b.full = dfixed_mul(b, c);
2135 lb_fill_bw = min(dfixed_trunc(a), dfixed_trunc(b));
2137 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2138 b.full = dfixed_const(1000);
2139 c.full = dfixed_const(lb_fill_bw);
2140 b.full = dfixed_div(c, b);
2141 a.full = dfixed_div(a, b);
2142 line_fill_time = dfixed_trunc(a);
2144 if (line_fill_time < wm->active_time)
2147 return latency + (line_fill_time - wm->active_time);
2151 static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2153 if (evergreen_average_bandwidth(wm) <=
2154 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2160 static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2162 if (evergreen_average_bandwidth(wm) <=
2163 (evergreen_available_bandwidth(wm) / wm->num_heads))
2169 static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2171 u32 lb_partitions = wm->lb_size / wm->src_width;
2172 u32 line_time = wm->active_time + wm->blank_time;
2173 u32 latency_tolerant_lines;
2177 a.full = dfixed_const(1);
2178 if (wm->vsc.full > a.full)
2179 latency_tolerant_lines = 1;
2181 if (lb_partitions <= (wm->vtaps + 1))
2182 latency_tolerant_lines = 1;
2184 latency_tolerant_lines = 2;
2187 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2189 if (evergreen_latency_watermark(wm) <= latency_hiding)
2195 static void evergreen_program_watermarks(struct radeon_device *rdev,
2196 struct radeon_crtc *radeon_crtc,
2197 u32 lb_size, u32 num_heads)
2199 struct drm_display_mode *mode = &radeon_crtc->base.mode;
2200 struct evergreen_wm_params wm_low, wm_high;
2204 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2205 u32 priority_a_mark = 0, priority_b_mark = 0;
2206 u32 priority_a_cnt = PRIORITY_OFF;
2207 u32 priority_b_cnt = PRIORITY_OFF;
2208 u32 pipe_offset = radeon_crtc->crtc_id * 16;
2209 u32 tmp, arb_control3;
2212 if (radeon_crtc->base.enabled && num_heads && mode) {
2213 pixel_period = 1000000 / (u32)mode->clock;
2214 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2217 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2219 /* watermark for high clocks */
2220 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2222 radeon_dpm_get_mclk(rdev, false) * 10;
2224 radeon_dpm_get_sclk(rdev, false) * 10;
2226 wm_high.yclk = rdev->pm.current_mclk * 10;
2227 wm_high.sclk = rdev->pm.current_sclk * 10;
2230 wm_high.disp_clk = mode->clock;
2231 wm_high.src_width = mode->crtc_hdisplay;
2232 wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2233 wm_high.blank_time = line_time - wm_high.active_time;
2234 wm_high.interlaced = false;
2235 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2236 wm_high.interlaced = true;
2237 wm_high.vsc = radeon_crtc->vsc;
2239 if (radeon_crtc->rmx_type != RMX_OFF)
2241 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2242 wm_high.lb_size = lb_size;
2243 wm_high.dram_channels = dram_channels;
2244 wm_high.num_heads = num_heads;
2246 /* watermark for low clocks */
2247 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2249 radeon_dpm_get_mclk(rdev, true) * 10;
2251 radeon_dpm_get_sclk(rdev, true) * 10;
2253 wm_low.yclk = rdev->pm.current_mclk * 10;
2254 wm_low.sclk = rdev->pm.current_sclk * 10;
2257 wm_low.disp_clk = mode->clock;
2258 wm_low.src_width = mode->crtc_hdisplay;
2259 wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2260 wm_low.blank_time = line_time - wm_low.active_time;
2261 wm_low.interlaced = false;
2262 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2263 wm_low.interlaced = true;
2264 wm_low.vsc = radeon_crtc->vsc;
2266 if (radeon_crtc->rmx_type != RMX_OFF)
2268 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2269 wm_low.lb_size = lb_size;
2270 wm_low.dram_channels = dram_channels;
2271 wm_low.num_heads = num_heads;
2273 /* set for high clocks */
2274 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
2275 /* set for low clocks */
2276 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
2278 /* possibly force display priority to high */
2279 /* should really do this at mode validation time... */
2280 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2281 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2282 !evergreen_check_latency_hiding(&wm_high) ||
2283 (rdev->disp_priority == 2)) {
2284 DRM_DEBUG_KMS("force priority a to high\n");
2285 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2287 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2288 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2289 !evergreen_check_latency_hiding(&wm_low) ||
2290 (rdev->disp_priority == 2)) {
2291 DRM_DEBUG_KMS("force priority b to high\n");
2292 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2295 a.full = dfixed_const(1000);
2296 b.full = dfixed_const(mode->clock);
2297 b.full = dfixed_div(b, a);
2298 c.full = dfixed_const(latency_watermark_a);
2299 c.full = dfixed_mul(c, b);
2300 c.full = dfixed_mul(c, radeon_crtc->hsc);
2301 c.full = dfixed_div(c, a);
2302 a.full = dfixed_const(16);
2303 c.full = dfixed_div(c, a);
2304 priority_a_mark = dfixed_trunc(c);
2305 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2307 a.full = dfixed_const(1000);
2308 b.full = dfixed_const(mode->clock);
2309 b.full = dfixed_div(b, a);
2310 c.full = dfixed_const(latency_watermark_b);
2311 c.full = dfixed_mul(c, b);
2312 c.full = dfixed_mul(c, radeon_crtc->hsc);
2313 c.full = dfixed_div(c, a);
2314 a.full = dfixed_const(16);
2315 c.full = dfixed_div(c, a);
2316 priority_b_mark = dfixed_trunc(c);
2317 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2321 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2323 tmp &= ~LATENCY_WATERMARK_MASK(3);
2324 tmp |= LATENCY_WATERMARK_MASK(1);
2325 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2326 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2327 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2328 LATENCY_HIGH_WATERMARK(line_time)));
2330 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2331 tmp &= ~LATENCY_WATERMARK_MASK(3);
2332 tmp |= LATENCY_WATERMARK_MASK(2);
2333 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2334 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2335 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2336 LATENCY_HIGH_WATERMARK(line_time)));
2337 /* restore original selection */
2338 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2340 /* write the priority marks */
2341 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2342 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2344 /* save values for DPM */
2345 radeon_crtc->line_time = line_time;
2346 radeon_crtc->wm_high = latency_watermark_a;
2347 radeon_crtc->wm_low = latency_watermark_b;
2351 * evergreen_bandwidth_update - update display watermarks callback.
2353 * @rdev: radeon_device pointer
2355 * Update the display watermarks based on the requested mode(s)
2358 void evergreen_bandwidth_update(struct radeon_device *rdev)
2360 struct drm_display_mode *mode0 = NULL;
2361 struct drm_display_mode *mode1 = NULL;
2362 u32 num_heads = 0, lb_size;
2365 if (!rdev->mode_info.mode_config_initialized)
2368 radeon_update_display_priority(rdev);
2370 for (i = 0; i < rdev->num_crtc; i++) {
2371 if (rdev->mode_info.crtcs[i]->base.enabled)
2374 for (i = 0; i < rdev->num_crtc; i += 2) {
2375 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2376 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2377 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2378 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2379 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2380 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2385 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2387 * @rdev: radeon_device pointer
2389 * Wait for the MC (memory controller) to be idle.
2391 * Returns 0 if the MC is idle, -1 if not.
2393 int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
2398 for (i = 0; i < rdev->usec_timeout; i++) {
2399 /* read MC_STATUS */
2400 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2411 void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2416 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2418 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2419 for (i = 0; i < rdev->usec_timeout; i++) {
2420 /* read MC_STATUS */
2421 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2422 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2424 printk(KERN_WARNING "[drm] r600 flush TLB failed\n");
2434 static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
2439 if (rdev->gart.robj == NULL) {
2440 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2443 r = radeon_gart_table_vram_pin(rdev);
2446 radeon_gart_restore(rdev);
2447 /* Setup L2 cache */
2448 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2449 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2450 EFFECTIVE_L2_QUEUE_SIZE(7));
2451 WREG32(VM_L2_CNTL2, 0);
2452 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2453 /* Setup TLB control */
2454 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2455 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2456 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2457 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2458 if (rdev->flags & RADEON_IS_IGP) {
2459 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2460 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2461 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2463 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2464 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2465 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2466 if ((rdev->family == CHIP_JUNIPER) ||
2467 (rdev->family == CHIP_CYPRESS) ||
2468 (rdev->family == CHIP_HEMLOCK) ||
2469 (rdev->family == CHIP_BARTS))
2470 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
2472 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2473 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2474 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2475 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2476 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2477 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2478 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2479 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2480 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2481 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2482 (u32)(rdev->dummy_page.addr >> 12));
2483 WREG32(VM_CONTEXT1_CNTL, 0);
2485 evergreen_pcie_gart_tlb_flush(rdev);
2486 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2487 (unsigned)(rdev->mc.gtt_size >> 20),
2488 (unsigned long long)rdev->gart.table_addr);
2489 rdev->gart.ready = true;
2493 static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
2497 /* Disable all tables */
2498 WREG32(VM_CONTEXT0_CNTL, 0);
2499 WREG32(VM_CONTEXT1_CNTL, 0);
2501 /* Setup L2 cache */
2502 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2503 EFFECTIVE_L2_QUEUE_SIZE(7));
2504 WREG32(VM_L2_CNTL2, 0);
2505 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2506 /* Setup TLB control */
2507 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2508 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2509 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2510 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2511 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2512 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2513 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2514 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2515 radeon_gart_table_vram_unpin(rdev);
2518 static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
2520 evergreen_pcie_gart_disable(rdev);
2521 radeon_gart_table_vram_free(rdev);
2522 radeon_gart_fini(rdev);
2526 static void evergreen_agp_enable(struct radeon_device *rdev)
2530 /* Setup L2 cache */
2531 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2532 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2533 EFFECTIVE_L2_QUEUE_SIZE(7));
2534 WREG32(VM_L2_CNTL2, 0);
2535 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2536 /* Setup TLB control */
2537 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2538 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2539 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2540 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2541 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2542 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2543 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2544 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2545 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2546 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2547 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2548 WREG32(VM_CONTEXT0_CNTL, 0);
2549 WREG32(VM_CONTEXT1_CNTL, 0);
2552 void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
2554 u32 crtc_enabled, tmp, frame_count, blackout;
2557 if (!ASIC_IS_NODCE(rdev)) {
2558 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2559 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
2561 /* disable VGA render */
2562 WREG32(VGA_RENDER_CONTROL, 0);
2564 /* blank the display controllers */
2565 for (i = 0; i < rdev->num_crtc; i++) {
2566 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2568 save->crtc_enabled[i] = true;
2569 if (ASIC_IS_DCE6(rdev)) {
2570 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2571 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2572 radeon_wait_for_vblank(rdev, i);
2573 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2574 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2575 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2576 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2579 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2580 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2581 radeon_wait_for_vblank(rdev, i);
2582 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2583 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2584 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2585 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2588 /* wait for the next frame */
2589 frame_count = radeon_get_vblank_counter(rdev, i);
2590 for (j = 0; j < rdev->usec_timeout; j++) {
2591 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2596 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2597 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2598 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2599 tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2600 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2601 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2602 save->crtc_enabled[i] = false;
2605 save->crtc_enabled[i] = false;
2609 radeon_mc_wait_for_idle(rdev);
2611 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2612 if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2613 /* Block CPU access */
2614 WREG32(BIF_FB_EN, 0);
2615 /* blackout the MC */
2616 blackout &= ~BLACKOUT_MODE_MASK;
2617 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
2619 /* wait for the MC to settle */
2622 /* lock double buffered regs */
2623 for (i = 0; i < rdev->num_crtc; i++) {
2624 if (save->crtc_enabled[i]) {
2625 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2626 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2627 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2628 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2630 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2633 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2639 void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
2641 u32 tmp, frame_count;
2644 /* update crtc base addresses */
2645 for (i = 0; i < rdev->num_crtc; i++) {
2646 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2647 upper_32_bits(rdev->mc.vram_start));
2648 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2649 upper_32_bits(rdev->mc.vram_start));
2650 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
2651 (u32)rdev->mc.vram_start);
2652 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
2653 (u32)rdev->mc.vram_start);
2656 if (!ASIC_IS_NODCE(rdev)) {
2657 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2658 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2661 /* unlock regs and wait for update */
2662 for (i = 0; i < rdev->num_crtc; i++) {
2663 if (save->crtc_enabled[i]) {
2664 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2665 if ((tmp & 0x3) != 0) {
2667 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2669 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2670 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2671 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2672 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2674 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2677 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2679 for (j = 0; j < rdev->usec_timeout; j++) {
2680 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2681 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2688 /* unblackout the MC */
2689 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2690 tmp &= ~BLACKOUT_MODE_MASK;
2691 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2692 /* allow CPU access */
2693 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2695 for (i = 0; i < rdev->num_crtc; i++) {
2696 if (save->crtc_enabled[i]) {
2697 if (ASIC_IS_DCE6(rdev)) {
2698 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2699 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2700 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2701 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2702 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2704 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2705 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2706 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2707 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2708 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2710 /* wait for the next frame */
2711 frame_count = radeon_get_vblank_counter(rdev, i);
2712 for (j = 0; j < rdev->usec_timeout; j++) {
2713 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2719 if (!ASIC_IS_NODCE(rdev)) {
2720 /* Unlock vga access */
2721 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2723 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2727 void evergreen_mc_program(struct radeon_device *rdev)
2729 struct evergreen_mc_save save;
2733 /* Initialize HDP */
2734 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2735 WREG32((0x2c14 + j), 0x00000000);
2736 WREG32((0x2c18 + j), 0x00000000);
2737 WREG32((0x2c1c + j), 0x00000000);
2738 WREG32((0x2c20 + j), 0x00000000);
2739 WREG32((0x2c24 + j), 0x00000000);
2741 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2743 evergreen_mc_stop(rdev, &save);
2744 if (evergreen_mc_wait_for_idle(rdev)) {
2745 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2747 /* Lockout access through VGA aperture*/
2748 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2749 /* Update configuration */
2750 if (rdev->flags & RADEON_IS_AGP) {
2751 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2752 /* VRAM before AGP */
2753 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2754 rdev->mc.vram_start >> 12);
2755 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2756 rdev->mc.gtt_end >> 12);
2758 /* VRAM after AGP */
2759 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2760 rdev->mc.gtt_start >> 12);
2761 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2762 rdev->mc.vram_end >> 12);
2765 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2766 rdev->mc.vram_start >> 12);
2767 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2768 rdev->mc.vram_end >> 12);
2770 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
2771 /* llano/ontario only */
2772 if ((rdev->family == CHIP_PALM) ||
2773 (rdev->family == CHIP_SUMO) ||
2774 (rdev->family == CHIP_SUMO2)) {
2775 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2776 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2777 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2778 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2780 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2781 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2782 WREG32(MC_VM_FB_LOCATION, tmp);
2783 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2784 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2785 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2786 if (rdev->flags & RADEON_IS_AGP) {
2787 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2788 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2789 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2791 WREG32(MC_VM_AGP_BASE, 0);
2792 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2793 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2795 if (evergreen_mc_wait_for_idle(rdev)) {
2796 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2798 evergreen_mc_resume(rdev, &save);
2799 /* we need to own VRAM, so turn off the VGA renderer here
2800 * to stop it overwriting our objects */
2801 rv515_vga_render_disable(rdev);
2807 void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2809 struct radeon_ring *ring = &rdev->ring[ib->ring];
2812 /* set to DX10/11 mode */
2813 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2814 radeon_ring_write(ring, 1);
2816 if (ring->rptr_save_reg) {
2817 next_rptr = ring->wptr + 3 + 4;
2818 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2819 radeon_ring_write(ring, ((ring->rptr_save_reg -
2820 PACKET3_SET_CONFIG_REG_START) >> 2));
2821 radeon_ring_write(ring, next_rptr);
2822 } else if (rdev->wb.enabled) {
2823 next_rptr = ring->wptr + 5 + 4;
2824 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2825 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2826 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2827 radeon_ring_write(ring, next_rptr);
2828 radeon_ring_write(ring, 0);
2831 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2832 radeon_ring_write(ring,
2836 (ib->gpu_addr & 0xFFFFFFFC));
2837 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2838 radeon_ring_write(ring, ib->length_dw);
2842 static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2844 const __be32 *fw_data;
2847 if (!rdev->me_fw || !rdev->pfp_fw)
2855 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2857 fw_data = (const __be32 *)rdev->pfp_fw->data;
2858 WREG32(CP_PFP_UCODE_ADDR, 0);
2859 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2860 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2861 WREG32(CP_PFP_UCODE_ADDR, 0);
2863 fw_data = (const __be32 *)rdev->me_fw->data;
2864 WREG32(CP_ME_RAM_WADDR, 0);
2865 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2866 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2868 WREG32(CP_PFP_UCODE_ADDR, 0);
2869 WREG32(CP_ME_RAM_WADDR, 0);
2870 WREG32(CP_ME_RAM_RADDR, 0);
2874 static int evergreen_cp_start(struct radeon_device *rdev)
2876 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2880 r = radeon_ring_lock(rdev, ring, 7);
2882 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2885 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2886 radeon_ring_write(ring, 0x1);
2887 radeon_ring_write(ring, 0x0);
2888 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
2889 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2890 radeon_ring_write(ring, 0);
2891 radeon_ring_write(ring, 0);
2892 radeon_ring_unlock_commit(rdev, ring);
2895 WREG32(CP_ME_CNTL, cp_me);
2897 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
2899 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2903 /* setup clear context state */
2904 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2905 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2907 for (i = 0; i < evergreen_default_size; i++)
2908 radeon_ring_write(ring, evergreen_default_state[i]);
2910 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2911 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2913 /* set clear context state */
2914 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2915 radeon_ring_write(ring, 0);
2917 /* SQ_VTX_BASE_VTX_LOC */
2918 radeon_ring_write(ring, 0xc0026f00);
2919 radeon_ring_write(ring, 0x00000000);
2920 radeon_ring_write(ring, 0x00000000);
2921 radeon_ring_write(ring, 0x00000000);
2924 radeon_ring_write(ring, 0xc0036f00);
2925 radeon_ring_write(ring, 0x00000bc4);
2926 radeon_ring_write(ring, 0xffffffff);
2927 radeon_ring_write(ring, 0xffffffff);
2928 radeon_ring_write(ring, 0xffffffff);
2930 radeon_ring_write(ring, 0xc0026900);
2931 radeon_ring_write(ring, 0x00000316);
2932 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2933 radeon_ring_write(ring, 0x00000010); /* */
2935 radeon_ring_unlock_commit(rdev, ring);
2940 static int evergreen_cp_resume(struct radeon_device *rdev)
2942 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2947 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
2948 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
2954 RREG32(GRBM_SOFT_RESET);
2956 WREG32(GRBM_SOFT_RESET, 0);
2957 RREG32(GRBM_SOFT_RESET);
2959 /* Set ring buffer size */
2960 rb_bufsz = order_base_2(ring->ring_size / 8);
2961 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2963 tmp |= BUF_SWAP_32BIT;
2965 WREG32(CP_RB_CNTL, tmp);
2966 WREG32(CP_SEM_WAIT_TIMER, 0x0);
2967 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
2969 /* Set the write pointer delay */
2970 WREG32(CP_RB_WPTR_DELAY, 0);
2972 /* Initialize the ring buffer's read and write pointers */
2973 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
2974 WREG32(CP_RB_RPTR_WR, 0);
2976 WREG32(CP_RB_WPTR, ring->wptr);
2978 /* set the wb address whether it's enabled or not */
2979 WREG32(CP_RB_RPTR_ADDR,
2980 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
2981 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2982 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2984 if (rdev->wb.enabled)
2985 WREG32(SCRATCH_UMSK, 0xff);
2987 tmp |= RB_NO_UPDATE;
2988 WREG32(SCRATCH_UMSK, 0);
2992 WREG32(CP_RB_CNTL, tmp);
2994 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
2995 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
2997 ring->rptr = RREG32(CP_RB_RPTR);
2999 evergreen_cp_start(rdev);
3001 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
3003 ring->ready = false;
3012 static void evergreen_gpu_init(struct radeon_device *rdev)
3015 u32 mc_shared_chmap, mc_arb_ramcfg;
3019 u32 sq_lds_resource_mgmt;
3020 u32 sq_gpr_resource_mgmt_1;
3021 u32 sq_gpr_resource_mgmt_2;
3022 u32 sq_gpr_resource_mgmt_3;
3023 u32 sq_thread_resource_mgmt;
3024 u32 sq_thread_resource_mgmt_2;
3025 u32 sq_stack_resource_mgmt_1;
3026 u32 sq_stack_resource_mgmt_2;
3027 u32 sq_stack_resource_mgmt_3;
3028 u32 vgt_cache_invalidation;
3029 u32 hdp_host_path_cntl, tmp;
3030 u32 disabled_rb_mask;
3031 int i, j, num_shader_engines, ps_thread_count;
3033 switch (rdev->family) {
3036 rdev->config.evergreen.num_ses = 2;
3037 rdev->config.evergreen.max_pipes = 4;
3038 rdev->config.evergreen.max_tile_pipes = 8;
3039 rdev->config.evergreen.max_simds = 10;
3040 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3041 rdev->config.evergreen.max_gprs = 256;
3042 rdev->config.evergreen.max_threads = 248;
3043 rdev->config.evergreen.max_gs_threads = 32;
3044 rdev->config.evergreen.max_stack_entries = 512;
3045 rdev->config.evergreen.sx_num_of_sets = 4;
3046 rdev->config.evergreen.sx_max_export_size = 256;
3047 rdev->config.evergreen.sx_max_export_pos_size = 64;
3048 rdev->config.evergreen.sx_max_export_smx_size = 192;
3049 rdev->config.evergreen.max_hw_contexts = 8;
3050 rdev->config.evergreen.sq_num_cf_insts = 2;
3052 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3053 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3054 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3055 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
3058 rdev->config.evergreen.num_ses = 1;
3059 rdev->config.evergreen.max_pipes = 4;
3060 rdev->config.evergreen.max_tile_pipes = 4;
3061 rdev->config.evergreen.max_simds = 10;
3062 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3063 rdev->config.evergreen.max_gprs = 256;
3064 rdev->config.evergreen.max_threads = 248;
3065 rdev->config.evergreen.max_gs_threads = 32;
3066 rdev->config.evergreen.max_stack_entries = 512;
3067 rdev->config.evergreen.sx_num_of_sets = 4;
3068 rdev->config.evergreen.sx_max_export_size = 256;
3069 rdev->config.evergreen.sx_max_export_pos_size = 64;
3070 rdev->config.evergreen.sx_max_export_smx_size = 192;
3071 rdev->config.evergreen.max_hw_contexts = 8;
3072 rdev->config.evergreen.sq_num_cf_insts = 2;
3074 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3075 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3076 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3077 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
3080 rdev->config.evergreen.num_ses = 1;
3081 rdev->config.evergreen.max_pipes = 4;
3082 rdev->config.evergreen.max_tile_pipes = 4;
3083 rdev->config.evergreen.max_simds = 5;
3084 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3085 rdev->config.evergreen.max_gprs = 256;
3086 rdev->config.evergreen.max_threads = 248;
3087 rdev->config.evergreen.max_gs_threads = 32;
3088 rdev->config.evergreen.max_stack_entries = 256;
3089 rdev->config.evergreen.sx_num_of_sets = 4;
3090 rdev->config.evergreen.sx_max_export_size = 256;
3091 rdev->config.evergreen.sx_max_export_pos_size = 64;
3092 rdev->config.evergreen.sx_max_export_smx_size = 192;
3093 rdev->config.evergreen.max_hw_contexts = 8;
3094 rdev->config.evergreen.sq_num_cf_insts = 2;
3096 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3097 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3098 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3099 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
3103 rdev->config.evergreen.num_ses = 1;
3104 rdev->config.evergreen.max_pipes = 2;
3105 rdev->config.evergreen.max_tile_pipes = 2;
3106 rdev->config.evergreen.max_simds = 2;
3107 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3108 rdev->config.evergreen.max_gprs = 256;
3109 rdev->config.evergreen.max_threads = 192;
3110 rdev->config.evergreen.max_gs_threads = 16;
3111 rdev->config.evergreen.max_stack_entries = 256;
3112 rdev->config.evergreen.sx_num_of_sets = 4;
3113 rdev->config.evergreen.sx_max_export_size = 128;
3114 rdev->config.evergreen.sx_max_export_pos_size = 32;
3115 rdev->config.evergreen.sx_max_export_smx_size = 96;
3116 rdev->config.evergreen.max_hw_contexts = 4;
3117 rdev->config.evergreen.sq_num_cf_insts = 1;
3119 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3120 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3121 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3122 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3125 rdev->config.evergreen.num_ses = 1;
3126 rdev->config.evergreen.max_pipes = 2;
3127 rdev->config.evergreen.max_tile_pipes = 2;
3128 rdev->config.evergreen.max_simds = 2;
3129 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3130 rdev->config.evergreen.max_gprs = 256;
3131 rdev->config.evergreen.max_threads = 192;
3132 rdev->config.evergreen.max_gs_threads = 16;
3133 rdev->config.evergreen.max_stack_entries = 256;
3134 rdev->config.evergreen.sx_num_of_sets = 4;
3135 rdev->config.evergreen.sx_max_export_size = 128;
3136 rdev->config.evergreen.sx_max_export_pos_size = 32;
3137 rdev->config.evergreen.sx_max_export_smx_size = 96;
3138 rdev->config.evergreen.max_hw_contexts = 4;
3139 rdev->config.evergreen.sq_num_cf_insts = 1;
3141 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3142 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3143 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3144 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3147 rdev->config.evergreen.num_ses = 1;
3148 rdev->config.evergreen.max_pipes = 4;
3149 rdev->config.evergreen.max_tile_pipes = 4;
3150 if (rdev->pdev->device == 0x9648)
3151 rdev->config.evergreen.max_simds = 3;
3152 else if ((rdev->pdev->device == 0x9647) ||
3153 (rdev->pdev->device == 0x964a))
3154 rdev->config.evergreen.max_simds = 4;
3156 rdev->config.evergreen.max_simds = 5;
3157 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3158 rdev->config.evergreen.max_gprs = 256;
3159 rdev->config.evergreen.max_threads = 248;
3160 rdev->config.evergreen.max_gs_threads = 32;
3161 rdev->config.evergreen.max_stack_entries = 256;
3162 rdev->config.evergreen.sx_num_of_sets = 4;
3163 rdev->config.evergreen.sx_max_export_size = 256;
3164 rdev->config.evergreen.sx_max_export_pos_size = 64;
3165 rdev->config.evergreen.sx_max_export_smx_size = 192;
3166 rdev->config.evergreen.max_hw_contexts = 8;
3167 rdev->config.evergreen.sq_num_cf_insts = 2;
3169 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3170 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3171 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3172 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
3175 rdev->config.evergreen.num_ses = 1;
3176 rdev->config.evergreen.max_pipes = 4;
3177 rdev->config.evergreen.max_tile_pipes = 4;
3178 rdev->config.evergreen.max_simds = 2;
3179 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3180 rdev->config.evergreen.max_gprs = 256;
3181 rdev->config.evergreen.max_threads = 248;
3182 rdev->config.evergreen.max_gs_threads = 32;
3183 rdev->config.evergreen.max_stack_entries = 512;
3184 rdev->config.evergreen.sx_num_of_sets = 4;
3185 rdev->config.evergreen.sx_max_export_size = 256;
3186 rdev->config.evergreen.sx_max_export_pos_size = 64;
3187 rdev->config.evergreen.sx_max_export_smx_size = 192;
3188 rdev->config.evergreen.max_hw_contexts = 4;
3189 rdev->config.evergreen.sq_num_cf_insts = 2;
3191 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3192 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3193 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3194 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
3197 rdev->config.evergreen.num_ses = 2;
3198 rdev->config.evergreen.max_pipes = 4;
3199 rdev->config.evergreen.max_tile_pipes = 8;
3200 rdev->config.evergreen.max_simds = 7;
3201 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3202 rdev->config.evergreen.max_gprs = 256;
3203 rdev->config.evergreen.max_threads = 248;
3204 rdev->config.evergreen.max_gs_threads = 32;
3205 rdev->config.evergreen.max_stack_entries = 512;
3206 rdev->config.evergreen.sx_num_of_sets = 4;
3207 rdev->config.evergreen.sx_max_export_size = 256;
3208 rdev->config.evergreen.sx_max_export_pos_size = 64;
3209 rdev->config.evergreen.sx_max_export_smx_size = 192;
3210 rdev->config.evergreen.max_hw_contexts = 8;
3211 rdev->config.evergreen.sq_num_cf_insts = 2;
3213 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3214 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3215 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3216 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
3219 rdev->config.evergreen.num_ses = 1;
3220 rdev->config.evergreen.max_pipes = 4;
3221 rdev->config.evergreen.max_tile_pipes = 4;
3222 rdev->config.evergreen.max_simds = 6;
3223 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3224 rdev->config.evergreen.max_gprs = 256;
3225 rdev->config.evergreen.max_threads = 248;
3226 rdev->config.evergreen.max_gs_threads = 32;
3227 rdev->config.evergreen.max_stack_entries = 256;
3228 rdev->config.evergreen.sx_num_of_sets = 4;
3229 rdev->config.evergreen.sx_max_export_size = 256;
3230 rdev->config.evergreen.sx_max_export_pos_size = 64;
3231 rdev->config.evergreen.sx_max_export_smx_size = 192;
3232 rdev->config.evergreen.max_hw_contexts = 8;
3233 rdev->config.evergreen.sq_num_cf_insts = 2;
3235 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3236 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3237 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3238 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
3241 rdev->config.evergreen.num_ses = 1;
3242 rdev->config.evergreen.max_pipes = 2;
3243 rdev->config.evergreen.max_tile_pipes = 2;
3244 rdev->config.evergreen.max_simds = 2;
3245 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3246 rdev->config.evergreen.max_gprs = 256;
3247 rdev->config.evergreen.max_threads = 192;
3248 rdev->config.evergreen.max_gs_threads = 16;
3249 rdev->config.evergreen.max_stack_entries = 256;
3250 rdev->config.evergreen.sx_num_of_sets = 4;
3251 rdev->config.evergreen.sx_max_export_size = 128;
3252 rdev->config.evergreen.sx_max_export_pos_size = 32;
3253 rdev->config.evergreen.sx_max_export_smx_size = 96;
3254 rdev->config.evergreen.max_hw_contexts = 4;
3255 rdev->config.evergreen.sq_num_cf_insts = 1;
3257 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3258 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3259 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3260 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
3264 /* Initialize HDP */
3265 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3266 WREG32((0x2c14 + j), 0x00000000);
3267 WREG32((0x2c18 + j), 0x00000000);
3268 WREG32((0x2c1c + j), 0x00000000);
3269 WREG32((0x2c20 + j), 0x00000000);
3270 WREG32((0x2c24 + j), 0x00000000);
3273 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3275 evergreen_fix_pci_max_read_req_size(rdev);
3277 mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3278 if ((rdev->family == CHIP_PALM) ||
3279 (rdev->family == CHIP_SUMO) ||
3280 (rdev->family == CHIP_SUMO2))
3281 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3283 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3285 /* setup tiling info dword. gb_addr_config is not adequate since it does
3286 * not have bank info, so create a custom tiling dword.
3287 * bits 3:0 num_pipes
3288 * bits 7:4 num_banks
3289 * bits 11:8 group_size
3290 * bits 15:12 row_size
3292 rdev->config.evergreen.tile_config = 0;
3293 switch (rdev->config.evergreen.max_tile_pipes) {
3296 rdev->config.evergreen.tile_config |= (0 << 0);
3299 rdev->config.evergreen.tile_config |= (1 << 0);
3302 rdev->config.evergreen.tile_config |= (2 << 0);
3305 rdev->config.evergreen.tile_config |= (3 << 0);
3308 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
3309 if (rdev->flags & RADEON_IS_IGP)
3310 rdev->config.evergreen.tile_config |= 1 << 4;
3312 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3313 case 0: /* four banks */
3314 rdev->config.evergreen.tile_config |= 0 << 4;
3316 case 1: /* eight banks */
3317 rdev->config.evergreen.tile_config |= 1 << 4;
3319 case 2: /* sixteen banks */
3321 rdev->config.evergreen.tile_config |= 2 << 4;
3325 rdev->config.evergreen.tile_config |= 0 << 8;
3326 rdev->config.evergreen.tile_config |=
3327 ((gb_addr_config & 0x30000000) >> 28) << 12;
3329 num_shader_engines = (gb_addr_config & NUM_SHADER_ENGINES(3) >> 12) + 1;
3331 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3335 efuse_straps_4 = RREG32_RCU(0x204);
3336 efuse_straps_3 = RREG32_RCU(0x203);
3337 tmp = (((efuse_straps_4 & 0xf) << 4) |
3338 ((efuse_straps_3 & 0xf0000000) >> 28));
3341 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3342 u32 rb_disable_bitmap;
3344 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3345 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3346 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3348 tmp |= rb_disable_bitmap;
3351 /* enabled rb are just the one not disabled :) */
3352 disabled_rb_mask = tmp;
3354 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3356 /* if all the backends are disabled, fix it up here */
3357 if ((disabled_rb_mask & tmp) == tmp) {
3358 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3359 disabled_rb_mask &= ~(1 << i);
3362 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3363 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3365 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3366 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3367 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3368 WREG32(DMA_TILING_CONFIG, gb_addr_config);
3369 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3370 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3371 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3373 if ((rdev->config.evergreen.max_backends == 1) &&
3374 (rdev->flags & RADEON_IS_IGP)) {
3375 if ((disabled_rb_mask & 3) == 1) {
3376 /* RB0 disabled, RB1 enabled */
3379 /* RB1 disabled, RB0 enabled */
3383 tmp = gb_addr_config & NUM_PIPES_MASK;
3384 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3385 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3387 WREG32(GB_BACKEND_MAP, tmp);
3389 WREG32(CGTS_SYS_TCC_DISABLE, 0);
3390 WREG32(CGTS_TCC_DISABLE, 0);
3391 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3392 WREG32(CGTS_USER_TCC_DISABLE, 0);
3394 /* set HW defaults for 3D engine */
3395 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3396 ROQ_IB2_START(0x2b)));
3398 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3400 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3405 sx_debug_1 = RREG32(SX_DEBUG_1);
3406 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3407 WREG32(SX_DEBUG_1, sx_debug_1);
3410 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3411 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3412 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3413 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3415 if (rdev->family <= CHIP_SUMO2)
3416 WREG32(SMX_SAR_CTL0, 0x00010000);
3418 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3419 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3420 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3422 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3423 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3424 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3426 WREG32(VGT_NUM_INSTANCES, 1);
3427 WREG32(SPI_CONFIG_CNTL, 0);
3428 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3429 WREG32(CP_PERFMON_CNTL, 0);
3431 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3432 FETCH_FIFO_HIWATER(0x4) |
3433 DONE_FIFO_HIWATER(0xe0) |
3434 ALU_UPDATE_FIFO_HIWATER(0x8)));
3436 sq_config = RREG32(SQ_CONFIG);
3437 sq_config &= ~(PS_PRIO(3) |
3441 sq_config |= (VC_ENABLE |
3448 switch (rdev->family) {
3454 /* no vertex cache */
3455 sq_config &= ~VC_ENABLE;
3461 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3463 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2))* 12 / 32);
3464 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3465 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3466 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3467 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3468 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3469 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3471 switch (rdev->family) {
3476 ps_thread_count = 96;
3479 ps_thread_count = 128;
3483 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
3484 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3485 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3486 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3487 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3488 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3490 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3491 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3492 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3493 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3494 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3495 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3497 WREG32(SQ_CONFIG, sq_config);
3498 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3499 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3500 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3501 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3502 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3503 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3504 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3505 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3506 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3507 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3509 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3510 FORCE_EOV_MAX_REZ_CNT(255)));
3512 switch (rdev->family) {
3518 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
3521 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
3524 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3525 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3527 WREG32(VGT_GS_VERTEX_REUSE, 16);
3528 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
3529 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3531 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3532 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3534 WREG32(CB_PERF_CTR0_SEL_0, 0);
3535 WREG32(CB_PERF_CTR0_SEL_1, 0);
3536 WREG32(CB_PERF_CTR1_SEL_0, 0);
3537 WREG32(CB_PERF_CTR1_SEL_1, 0);
3538 WREG32(CB_PERF_CTR2_SEL_0, 0);
3539 WREG32(CB_PERF_CTR2_SEL_1, 0);
3540 WREG32(CB_PERF_CTR3_SEL_0, 0);
3541 WREG32(CB_PERF_CTR3_SEL_1, 0);
3543 /* clear render buffer base addresses */
3544 WREG32(CB_COLOR0_BASE, 0);
3545 WREG32(CB_COLOR1_BASE, 0);
3546 WREG32(CB_COLOR2_BASE, 0);
3547 WREG32(CB_COLOR3_BASE, 0);
3548 WREG32(CB_COLOR4_BASE, 0);
3549 WREG32(CB_COLOR5_BASE, 0);
3550 WREG32(CB_COLOR6_BASE, 0);
3551 WREG32(CB_COLOR7_BASE, 0);
3552 WREG32(CB_COLOR8_BASE, 0);
3553 WREG32(CB_COLOR9_BASE, 0);
3554 WREG32(CB_COLOR10_BASE, 0);
3555 WREG32(CB_COLOR11_BASE, 0);
3557 /* set the shader const cache sizes to 0 */
3558 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3560 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3563 tmp = RREG32(HDP_MISC_CNTL);
3564 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3565 WREG32(HDP_MISC_CNTL, tmp);
3567 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3568 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3570 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3576 int evergreen_mc_init(struct radeon_device *rdev)
3579 int chansize, numchan;
3581 /* Get VRAM informations */
3582 rdev->mc.vram_is_ddr = true;
3583 if ((rdev->family == CHIP_PALM) ||
3584 (rdev->family == CHIP_SUMO) ||
3585 (rdev->family == CHIP_SUMO2))
3586 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3588 tmp = RREG32(MC_ARB_RAMCFG);
3589 if (tmp & CHANSIZE_OVERRIDE) {
3591 } else if (tmp & CHANSIZE_MASK) {
3596 tmp = RREG32(MC_SHARED_CHMAP);
3597 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3612 rdev->mc.vram_width = numchan * chansize;
3613 /* Could aper size report 0 ? */
3614 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3615 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3616 /* Setup GPU memory space */
3617 if ((rdev->family == CHIP_PALM) ||
3618 (rdev->family == CHIP_SUMO) ||
3619 (rdev->family == CHIP_SUMO2)) {
3620 /* size in bytes on fusion */
3621 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3622 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3624 /* size in MB on evergreen/cayman/tn */
3625 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3626 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3628 rdev->mc.visible_vram_size = rdev->mc.aper_size;
3629 r700_vram_gtt_location(rdev, &rdev->mc);
3630 radeon_update_bandwidth_info(rdev);
3635 void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
3637 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
3638 RREG32(GRBM_STATUS));
3639 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
3640 RREG32(GRBM_STATUS_SE0));
3641 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
3642 RREG32(GRBM_STATUS_SE1));
3643 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
3644 RREG32(SRBM_STATUS));
3645 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
3646 RREG32(SRBM_STATUS2));
3647 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3648 RREG32(CP_STALLED_STAT1));
3649 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3650 RREG32(CP_STALLED_STAT2));
3651 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
3652 RREG32(CP_BUSY_STAT));
3653 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
3655 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
3656 RREG32(DMA_STATUS_REG));
3657 if (rdev->family >= CHIP_CAYMAN) {
3658 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
3659 RREG32(DMA_STATUS_REG + 0x800));
3663 bool evergreen_is_display_hung(struct radeon_device *rdev)
3669 for (i = 0; i < rdev->num_crtc; i++) {
3670 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3671 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3672 crtc_hung |= (1 << i);
3676 for (j = 0; j < 10; j++) {
3677 for (i = 0; i < rdev->num_crtc; i++) {
3678 if (crtc_hung & (1 << i)) {
3679 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3680 if (tmp != crtc_status[i])
3681 crtc_hung &= ~(1 << i);
3692 u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3698 tmp = RREG32(GRBM_STATUS);
3699 if (tmp & (PA_BUSY | SC_BUSY |
3701 TA_BUSY | VGT_BUSY |
3703 SPI_BUSY | VGT_BUSY_NO_DMA))
3704 reset_mask |= RADEON_RESET_GFX;
3706 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3707 CP_BUSY | CP_COHERENCY_BUSY))
3708 reset_mask |= RADEON_RESET_CP;
3710 if (tmp & GRBM_EE_BUSY)
3711 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3713 /* DMA_STATUS_REG */
3714 tmp = RREG32(DMA_STATUS_REG);
3715 if (!(tmp & DMA_IDLE))
3716 reset_mask |= RADEON_RESET_DMA;
3719 tmp = RREG32(SRBM_STATUS2);
3721 reset_mask |= RADEON_RESET_DMA;
3724 tmp = RREG32(SRBM_STATUS);
3725 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3726 reset_mask |= RADEON_RESET_RLC;
3729 reset_mask |= RADEON_RESET_IH;
3732 reset_mask |= RADEON_RESET_SEM;
3734 if (tmp & GRBM_RQ_PENDING)
3735 reset_mask |= RADEON_RESET_GRBM;
3738 reset_mask |= RADEON_RESET_VMC;
3740 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3741 MCC_BUSY | MCD_BUSY))
3742 reset_mask |= RADEON_RESET_MC;
3744 if (evergreen_is_display_hung(rdev))
3745 reset_mask |= RADEON_RESET_DISPLAY;
3748 tmp = RREG32(VM_L2_STATUS);
3750 reset_mask |= RADEON_RESET_VMC;
3752 /* Skip MC reset as it's mostly likely not hung, just busy */
3753 if (reset_mask & RADEON_RESET_MC) {
3754 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3755 reset_mask &= ~RADEON_RESET_MC;
3761 static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3763 struct evergreen_mc_save save;
3764 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3767 if (reset_mask == 0)
3770 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3772 evergreen_print_gpu_status_regs(rdev);
3774 /* Disable CP parsing/prefetching */
3775 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3777 if (reset_mask & RADEON_RESET_DMA) {
3779 tmp = RREG32(DMA_RB_CNTL);
3780 tmp &= ~DMA_RB_ENABLE;
3781 WREG32(DMA_RB_CNTL, tmp);
3786 evergreen_mc_stop(rdev, &save);
3787 if (evergreen_mc_wait_for_idle(rdev)) {
3788 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3791 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3792 grbm_soft_reset |= SOFT_RESET_DB |
3805 if (reset_mask & RADEON_RESET_CP) {
3806 grbm_soft_reset |= SOFT_RESET_CP |
3809 srbm_soft_reset |= SOFT_RESET_GRBM;
3812 if (reset_mask & RADEON_RESET_DMA)
3813 srbm_soft_reset |= SOFT_RESET_DMA;
3815 if (reset_mask & RADEON_RESET_DISPLAY)
3816 srbm_soft_reset |= SOFT_RESET_DC;
3818 if (reset_mask & RADEON_RESET_RLC)
3819 srbm_soft_reset |= SOFT_RESET_RLC;
3821 if (reset_mask & RADEON_RESET_SEM)
3822 srbm_soft_reset |= SOFT_RESET_SEM;
3824 if (reset_mask & RADEON_RESET_IH)
3825 srbm_soft_reset |= SOFT_RESET_IH;
3827 if (reset_mask & RADEON_RESET_GRBM)
3828 srbm_soft_reset |= SOFT_RESET_GRBM;
3830 if (reset_mask & RADEON_RESET_VMC)
3831 srbm_soft_reset |= SOFT_RESET_VMC;
3833 if (!(rdev->flags & RADEON_IS_IGP)) {
3834 if (reset_mask & RADEON_RESET_MC)
3835 srbm_soft_reset |= SOFT_RESET_MC;
3838 if (grbm_soft_reset) {
3839 tmp = RREG32(GRBM_SOFT_RESET);
3840 tmp |= grbm_soft_reset;
3841 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3842 WREG32(GRBM_SOFT_RESET, tmp);
3843 tmp = RREG32(GRBM_SOFT_RESET);
3847 tmp &= ~grbm_soft_reset;
3848 WREG32(GRBM_SOFT_RESET, tmp);
3849 tmp = RREG32(GRBM_SOFT_RESET);
3852 if (srbm_soft_reset) {
3853 tmp = RREG32(SRBM_SOFT_RESET);
3854 tmp |= srbm_soft_reset;
3855 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3856 WREG32(SRBM_SOFT_RESET, tmp);
3857 tmp = RREG32(SRBM_SOFT_RESET);
3861 tmp &= ~srbm_soft_reset;
3862 WREG32(SRBM_SOFT_RESET, tmp);
3863 tmp = RREG32(SRBM_SOFT_RESET);
3866 /* Wait a little for things to settle down */
3869 evergreen_mc_resume(rdev, &save);
3872 evergreen_print_gpu_status_regs(rdev);
3875 void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)
3877 struct evergreen_mc_save save;
3880 dev_info(rdev->dev, "GPU pci config reset\n");
3884 /* Disable CP parsing/prefetching */
3885 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3888 tmp = RREG32(DMA_RB_CNTL);
3889 tmp &= ~DMA_RB_ENABLE;
3890 WREG32(DMA_RB_CNTL, tmp);
3891 /* XXX other engines? */
3894 r600_rlc_stop(rdev);
3898 /* set mclk/sclk to bypass */
3899 rv770_set_clk_bypass_mode(rdev);
3901 pci_clear_master(rdev->pdev);
3902 /* disable mem access */
3903 evergreen_mc_stop(rdev, &save);
3904 if (evergreen_mc_wait_for_idle(rdev)) {
3905 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
3908 radeon_pci_config_reset(rdev);
3909 /* wait for asic to come out of reset */
3910 for (i = 0; i < rdev->usec_timeout; i++) {
3911 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
3917 int evergreen_asic_reset(struct radeon_device *rdev)
3921 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3924 r600_set_bios_scratch_engine_hung(rdev, true);
3926 /* try soft reset */
3927 evergreen_gpu_soft_reset(rdev, reset_mask);
3929 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3931 /* try pci config reset */
3932 if (reset_mask && radeon_hard_reset)
3933 evergreen_gpu_pci_config_reset(rdev);
3935 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3938 r600_set_bios_scratch_engine_hung(rdev, false);
3944 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
3946 * @rdev: radeon_device pointer
3947 * @ring: radeon_ring structure holding ring information
3949 * Check if the GFX engine is locked up.
3950 * Returns true if the engine appears to be locked up, false if not.
3952 bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3954 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
3956 if (!(reset_mask & (RADEON_RESET_GFX |
3957 RADEON_RESET_COMPUTE |
3958 RADEON_RESET_CP))) {
3959 radeon_ring_lockup_update(ring);
3962 /* force CP activities */
3963 radeon_ring_force_activity(rdev, ring);
3964 return radeon_ring_test_lockup(rdev, ring);
3970 #define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000
3971 #define RLC_CLEAR_STATE_END_MARKER 0x00000001
3973 void sumo_rlc_fini(struct radeon_device *rdev)
3977 /* save restore block */
3978 if (rdev->rlc.save_restore_obj) {
3979 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
3980 if (unlikely(r != 0))
3981 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
3982 radeon_bo_unpin(rdev->rlc.save_restore_obj);
3983 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
3985 radeon_bo_unref(&rdev->rlc.save_restore_obj);
3986 rdev->rlc.save_restore_obj = NULL;
3989 /* clear state block */
3990 if (rdev->rlc.clear_state_obj) {
3991 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
3992 if (unlikely(r != 0))
3993 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
3994 radeon_bo_unpin(rdev->rlc.clear_state_obj);
3995 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
3997 radeon_bo_unref(&rdev->rlc.clear_state_obj);
3998 rdev->rlc.clear_state_obj = NULL;
4001 /* clear state block */
4002 if (rdev->rlc.cp_table_obj) {
4003 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4004 if (unlikely(r != 0))
4005 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4006 radeon_bo_unpin(rdev->rlc.cp_table_obj);
4007 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4009 radeon_bo_unref(&rdev->rlc.cp_table_obj);
4010 rdev->rlc.cp_table_obj = NULL;
4014 #define CP_ME_TABLE_SIZE 96
4016 int sumo_rlc_init(struct radeon_device *rdev)
4019 volatile u32 *dst_ptr;
4020 u32 dws, data, i, j, k, reg_num;
4021 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
4022 u64 reg_list_mc_addr;
4023 const struct cs_section_def *cs_data;
4026 src_ptr = rdev->rlc.reg_list;
4027 dws = rdev->rlc.reg_list_size;
4028 if (rdev->family >= CHIP_BONAIRE) {
4029 dws += (5 * 16) + 48 + 48 + 64;
4031 cs_data = rdev->rlc.cs_data;
4034 /* save restore block */
4035 if (rdev->rlc.save_restore_obj == NULL) {
4036 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4037 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.save_restore_obj);
4039 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4044 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4045 if (unlikely(r != 0)) {
4046 sumo_rlc_fini(rdev);
4049 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4050 &rdev->rlc.save_restore_gpu_addr);
4052 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4053 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4054 sumo_rlc_fini(rdev);
4058 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
4060 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4061 sumo_rlc_fini(rdev);
4064 /* write the sr buffer */
4065 dst_ptr = rdev->rlc.sr_ptr;
4066 if (rdev->family >= CHIP_TAHITI) {
4068 for (i = 0; i < rdev->rlc.reg_list_size; i++)
4069 dst_ptr[i] = cpu_to_le32(src_ptr[i]);
4073 * dw0: (reg2 << 16) | reg1
4074 * dw1: reg1 save space
4075 * dw2: reg2 save space
4077 for (i = 0; i < dws; i++) {
4078 data = src_ptr[i] >> 2;
4081 data |= (src_ptr[i] >> 2) << 16;
4082 j = (((i - 1) * 3) / 2);
4083 dst_ptr[j] = cpu_to_le32(data);
4086 dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
4088 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
4089 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4093 /* clear state block */
4094 if (rdev->family >= CHIP_BONAIRE) {
4095 rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4096 } else if (rdev->family >= CHIP_TAHITI) {
4097 rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4098 dws = rdev->rlc.clear_state_size + (256 / 4);
4102 for (i = 0; cs_data[i].section != NULL; i++) {
4103 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4105 dws += cs_data[i].section[j].reg_count;
4108 reg_list_blk_index = (3 * reg_list_num + 2);
4109 dws += reg_list_blk_index;
4110 rdev->rlc.clear_state_size = dws;
4113 if (rdev->rlc.clear_state_obj == NULL) {
4114 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4115 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.clear_state_obj);
4117 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4118 sumo_rlc_fini(rdev);
4122 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4123 if (unlikely(r != 0)) {
4124 sumo_rlc_fini(rdev);
4127 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4128 &rdev->rlc.clear_state_gpu_addr);
4130 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4131 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4132 sumo_rlc_fini(rdev);
4136 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4138 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4139 sumo_rlc_fini(rdev);
4142 /* set up the cs buffer */
4143 dst_ptr = rdev->rlc.cs_ptr;
4144 if (rdev->family >= CHIP_BONAIRE) {
4145 cik_get_csb_buffer(rdev, dst_ptr);
4146 } else if (rdev->family >= CHIP_TAHITI) {
4147 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
4148 dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
4149 dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
4150 dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
4151 si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4153 reg_list_hdr_blk_index = 0;
4154 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4155 data = upper_32_bits(reg_list_mc_addr);
4156 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4157 reg_list_hdr_blk_index++;
4158 for (i = 0; cs_data[i].section != NULL; i++) {
4159 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4160 reg_num = cs_data[i].section[j].reg_count;
4161 data = reg_list_mc_addr & 0xffffffff;
4162 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4163 reg_list_hdr_blk_index++;
4165 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4166 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4167 reg_list_hdr_blk_index++;
4169 data = 0x08000000 | (reg_num * 4);
4170 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4171 reg_list_hdr_blk_index++;
4173 for (k = 0; k < reg_num; k++) {
4174 data = cs_data[i].section[j].extent[k];
4175 dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
4177 reg_list_mc_addr += reg_num * 4;
4178 reg_list_blk_index += reg_num;
4181 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
4183 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4184 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4187 if (rdev->rlc.cp_table_size) {
4188 if (rdev->rlc.cp_table_obj == NULL) {
4189 r = radeon_bo_create(rdev, rdev->rlc.cp_table_size, PAGE_SIZE, true,
4190 RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->rlc.cp_table_obj);
4192 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4193 sumo_rlc_fini(rdev);
4198 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4199 if (unlikely(r != 0)) {
4200 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4201 sumo_rlc_fini(rdev);
4204 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4205 &rdev->rlc.cp_table_gpu_addr);
4207 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4208 dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4209 sumo_rlc_fini(rdev);
4212 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4214 dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4215 sumo_rlc_fini(rdev);
4219 cik_init_cp_pg_table(rdev);
4221 radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4222 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4229 static void evergreen_rlc_start(struct radeon_device *rdev)
4231 u32 mask = RLC_ENABLE;
4233 if (rdev->flags & RADEON_IS_IGP) {
4234 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
4237 WREG32(RLC_CNTL, mask);
4240 int evergreen_rlc_resume(struct radeon_device *rdev)
4243 const __be32 *fw_data;
4248 r600_rlc_stop(rdev);
4250 WREG32(RLC_HB_CNTL, 0);
4252 if (rdev->flags & RADEON_IS_IGP) {
4253 if (rdev->family == CHIP_ARUBA) {
4254 u32 always_on_bitmap =
4255 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4256 /* find out the number of active simds */
4257 u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4258 tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4259 tmp = hweight32(~tmp);
4260 if (tmp == rdev->config.cayman.max_simds_per_se) {
4261 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4262 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4263 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4264 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4265 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4268 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4269 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4271 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4272 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4274 WREG32(RLC_HB_BASE, 0);
4275 WREG32(RLC_HB_RPTR, 0);
4276 WREG32(RLC_HB_WPTR, 0);
4277 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4278 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4280 WREG32(RLC_MC_CNTL, 0);
4281 WREG32(RLC_UCODE_CNTL, 0);
4283 fw_data = (const __be32 *)rdev->rlc_fw->data;
4284 if (rdev->family >= CHIP_ARUBA) {
4285 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4286 WREG32(RLC_UCODE_ADDR, i);
4287 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4289 } else if (rdev->family >= CHIP_CAYMAN) {
4290 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4291 WREG32(RLC_UCODE_ADDR, i);
4292 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4295 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4296 WREG32(RLC_UCODE_ADDR, i);
4297 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4300 WREG32(RLC_UCODE_ADDR, 0);
4302 evergreen_rlc_start(rdev);
4309 u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4311 if (crtc >= rdev->num_crtc)
4314 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
4317 void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4321 if (rdev->family >= CHIP_CAYMAN) {
4322 cayman_cp_int_cntl_setup(rdev, 0,
4323 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4324 cayman_cp_int_cntl_setup(rdev, 1, 0);
4325 cayman_cp_int_cntl_setup(rdev, 2, 0);
4326 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4327 WREG32(CAYMAN_DMA1_CNTL, tmp);
4329 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4330 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4331 WREG32(DMA_CNTL, tmp);
4332 WREG32(GRBM_INT_CNTL, 0);
4333 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4334 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4335 if (rdev->num_crtc >= 4) {
4336 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4337 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4339 if (rdev->num_crtc >= 6) {
4340 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4341 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4344 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4345 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4346 if (rdev->num_crtc >= 4) {
4347 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4348 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4350 if (rdev->num_crtc >= 6) {
4351 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4352 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4355 /* only one DAC on DCE5 */
4356 if (!ASIC_IS_DCE5(rdev))
4357 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4358 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4360 tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4361 WREG32(DC_HPD1_INT_CONTROL, tmp);
4362 tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4363 WREG32(DC_HPD2_INT_CONTROL, tmp);
4364 tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4365 WREG32(DC_HPD3_INT_CONTROL, tmp);
4366 tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4367 WREG32(DC_HPD4_INT_CONTROL, tmp);
4368 tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4369 WREG32(DC_HPD5_INT_CONTROL, tmp);
4370 tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4371 WREG32(DC_HPD6_INT_CONTROL, tmp);
4375 int evergreen_irq_set(struct radeon_device *rdev)
4377 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4378 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4379 u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
4380 u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
4381 u32 grbm_int_cntl = 0;
4382 u32 afmt1 = 0, afmt2 = 0, afmt3 = 0, afmt4 = 0, afmt5 = 0, afmt6 = 0;
4383 u32 dma_cntl, dma_cntl1 = 0;
4384 u32 thermal_int = 0;
4386 if (!rdev->irq.installed) {
4387 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4390 /* don't enable anything if the ih is disabled */
4391 if (!rdev->ih.enabled) {
4392 r600_disable_interrupts(rdev);
4393 /* force the active interrupt state to all disabled */
4394 evergreen_disable_interrupt_state(rdev);
4398 hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
4399 hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
4400 hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
4401 hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
4402 hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
4403 hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
4404 if (rdev->family == CHIP_ARUBA)
4405 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4406 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4408 thermal_int = RREG32(CG_THERMAL_INT) &
4409 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4411 afmt1 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4412 afmt2 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4413 afmt3 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4414 afmt4 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4415 afmt5 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4416 afmt6 = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET) & ~AFMT_AZ_FORMAT_WTRIG_MASK;
4418 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4420 if (rdev->family >= CHIP_CAYMAN) {
4421 /* enable CP interrupts on all rings */
4422 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4423 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4424 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4426 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4427 DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4428 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4430 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4431 DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4432 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4435 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4436 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4437 cp_int_cntl |= RB_INT_ENABLE;
4438 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4442 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4443 DRM_DEBUG("r600_irq_set: sw int dma\n");
4444 dma_cntl |= TRAP_ENABLE;
4447 if (rdev->family >= CHIP_CAYMAN) {
4448 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4449 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4450 DRM_DEBUG("r600_irq_set: sw int dma1\n");
4451 dma_cntl1 |= TRAP_ENABLE;
4455 if (rdev->irq.dpm_thermal) {
4456 DRM_DEBUG("dpm thermal\n");
4457 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4460 if (rdev->irq.crtc_vblank_int[0] ||
4461 atomic_read(&rdev->irq.pflip[0])) {
4462 DRM_DEBUG("evergreen_irq_set: vblank 0\n");
4463 crtc1 |= VBLANK_INT_MASK;
4465 if (rdev->irq.crtc_vblank_int[1] ||
4466 atomic_read(&rdev->irq.pflip[1])) {
4467 DRM_DEBUG("evergreen_irq_set: vblank 1\n");
4468 crtc2 |= VBLANK_INT_MASK;
4470 if (rdev->irq.crtc_vblank_int[2] ||
4471 atomic_read(&rdev->irq.pflip[2])) {
4472 DRM_DEBUG("evergreen_irq_set: vblank 2\n");
4473 crtc3 |= VBLANK_INT_MASK;
4475 if (rdev->irq.crtc_vblank_int[3] ||
4476 atomic_read(&rdev->irq.pflip[3])) {
4477 DRM_DEBUG("evergreen_irq_set: vblank 3\n");
4478 crtc4 |= VBLANK_INT_MASK;
4480 if (rdev->irq.crtc_vblank_int[4] ||
4481 atomic_read(&rdev->irq.pflip[4])) {
4482 DRM_DEBUG("evergreen_irq_set: vblank 4\n");
4483 crtc5 |= VBLANK_INT_MASK;
4485 if (rdev->irq.crtc_vblank_int[5] ||
4486 atomic_read(&rdev->irq.pflip[5])) {
4487 DRM_DEBUG("evergreen_irq_set: vblank 5\n");
4488 crtc6 |= VBLANK_INT_MASK;
4490 if (rdev->irq.hpd[0]) {
4491 DRM_DEBUG("evergreen_irq_set: hpd 1\n");
4492 hpd1 |= DC_HPDx_INT_EN;
4494 if (rdev->irq.hpd[1]) {
4495 DRM_DEBUG("evergreen_irq_set: hpd 2\n");
4496 hpd2 |= DC_HPDx_INT_EN;
4498 if (rdev->irq.hpd[2]) {
4499 DRM_DEBUG("evergreen_irq_set: hpd 3\n");
4500 hpd3 |= DC_HPDx_INT_EN;
4502 if (rdev->irq.hpd[3]) {
4503 DRM_DEBUG("evergreen_irq_set: hpd 4\n");
4504 hpd4 |= DC_HPDx_INT_EN;
4506 if (rdev->irq.hpd[4]) {
4507 DRM_DEBUG("evergreen_irq_set: hpd 5\n");
4508 hpd5 |= DC_HPDx_INT_EN;
4510 if (rdev->irq.hpd[5]) {
4511 DRM_DEBUG("evergreen_irq_set: hpd 6\n");
4512 hpd6 |= DC_HPDx_INT_EN;
4514 if (rdev->irq.afmt[0]) {
4515 DRM_DEBUG("evergreen_irq_set: hdmi 0\n");
4516 afmt1 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4518 if (rdev->irq.afmt[1]) {
4519 DRM_DEBUG("evergreen_irq_set: hdmi 1\n");
4520 afmt2 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4522 if (rdev->irq.afmt[2]) {
4523 DRM_DEBUG("evergreen_irq_set: hdmi 2\n");
4524 afmt3 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4526 if (rdev->irq.afmt[3]) {
4527 DRM_DEBUG("evergreen_irq_set: hdmi 3\n");
4528 afmt4 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4530 if (rdev->irq.afmt[4]) {
4531 DRM_DEBUG("evergreen_irq_set: hdmi 4\n");
4532 afmt5 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4534 if (rdev->irq.afmt[5]) {
4535 DRM_DEBUG("evergreen_irq_set: hdmi 5\n");
4536 afmt6 |= AFMT_AZ_FORMAT_WTRIG_MASK;
4539 if (rdev->family >= CHIP_CAYMAN) {
4540 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4541 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4542 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4544 WREG32(CP_INT_CNTL, cp_int_cntl);
4546 WREG32(DMA_CNTL, dma_cntl);
4548 if (rdev->family >= CHIP_CAYMAN)
4549 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4551 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4553 WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4554 WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
4555 if (rdev->num_crtc >= 4) {
4556 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4557 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
4559 if (rdev->num_crtc >= 6) {
4560 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4561 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4564 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
4565 GRPH_PFLIP_INT_MASK);
4566 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
4567 GRPH_PFLIP_INT_MASK);
4568 if (rdev->num_crtc >= 4) {
4569 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
4570 GRPH_PFLIP_INT_MASK);
4571 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
4572 GRPH_PFLIP_INT_MASK);
4574 if (rdev->num_crtc >= 6) {
4575 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
4576 GRPH_PFLIP_INT_MASK);
4577 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
4578 GRPH_PFLIP_INT_MASK);
4581 WREG32(DC_HPD1_INT_CONTROL, hpd1);
4582 WREG32(DC_HPD2_INT_CONTROL, hpd2);
4583 WREG32(DC_HPD3_INT_CONTROL, hpd3);
4584 WREG32(DC_HPD4_INT_CONTROL, hpd4);
4585 WREG32(DC_HPD5_INT_CONTROL, hpd5);
4586 WREG32(DC_HPD6_INT_CONTROL, hpd6);
4587 if (rdev->family == CHIP_ARUBA)
4588 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4590 WREG32(CG_THERMAL_INT, thermal_int);
4592 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, afmt1);
4593 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, afmt2);
4594 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, afmt3);
4595 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, afmt4);
4596 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, afmt5);
4597 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, afmt6);
4602 static void evergreen_irq_ack(struct radeon_device *rdev)
4606 rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4607 rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4608 rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4609 rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4610 rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4611 rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4612 rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4613 rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4614 if (rdev->num_crtc >= 4) {
4615 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4616 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4618 if (rdev->num_crtc >= 6) {
4619 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4620 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4623 rdev->irq.stat_regs.evergreen.afmt_status1 = RREG32(AFMT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4624 rdev->irq.stat_regs.evergreen.afmt_status2 = RREG32(AFMT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4625 rdev->irq.stat_regs.evergreen.afmt_status3 = RREG32(AFMT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4626 rdev->irq.stat_regs.evergreen.afmt_status4 = RREG32(AFMT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4627 rdev->irq.stat_regs.evergreen.afmt_status5 = RREG32(AFMT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4628 rdev->irq.stat_regs.evergreen.afmt_status6 = RREG32(AFMT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4630 if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4631 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4632 if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4633 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4634 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
4635 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
4636 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
4637 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
4638 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
4639 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
4640 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
4641 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4643 if (rdev->num_crtc >= 4) {
4644 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4645 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4646 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4647 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4648 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4649 WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4650 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4651 WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4652 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4653 WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4654 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4655 WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4658 if (rdev->num_crtc >= 6) {
4659 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4660 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4661 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4662 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4663 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4664 WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4665 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4666 WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4667 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4668 WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4669 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4670 WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4673 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4674 tmp = RREG32(DC_HPD1_INT_CONTROL);
4675 tmp |= DC_HPDx_INT_ACK;
4676 WREG32(DC_HPD1_INT_CONTROL, tmp);
4678 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4679 tmp = RREG32(DC_HPD2_INT_CONTROL);
4680 tmp |= DC_HPDx_INT_ACK;
4681 WREG32(DC_HPD2_INT_CONTROL, tmp);
4683 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4684 tmp = RREG32(DC_HPD3_INT_CONTROL);
4685 tmp |= DC_HPDx_INT_ACK;
4686 WREG32(DC_HPD3_INT_CONTROL, tmp);
4688 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4689 tmp = RREG32(DC_HPD4_INT_CONTROL);
4690 tmp |= DC_HPDx_INT_ACK;
4691 WREG32(DC_HPD4_INT_CONTROL, tmp);
4693 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4694 tmp = RREG32(DC_HPD5_INT_CONTROL);
4695 tmp |= DC_HPDx_INT_ACK;
4696 WREG32(DC_HPD5_INT_CONTROL, tmp);
4698 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4699 tmp = RREG32(DC_HPD5_INT_CONTROL);
4700 tmp |= DC_HPDx_INT_ACK;
4701 WREG32(DC_HPD6_INT_CONTROL, tmp);
4703 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
4704 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET);
4705 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4706 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, tmp);
4708 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
4709 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET);
4710 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4711 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, tmp);
4713 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
4714 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET);
4715 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4716 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, tmp);
4718 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
4719 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET);
4720 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4721 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, tmp);
4723 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
4724 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET);
4725 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4726 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, tmp);
4728 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
4729 tmp = RREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET);
4730 tmp |= AFMT_AZ_FORMAT_WTRIG_ACK;
4731 WREG32(AFMT_AUDIO_PACKET_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, tmp);
4735 static void evergreen_irq_disable(struct radeon_device *rdev)
4737 r600_disable_interrupts(rdev);
4738 /* Wait and acknowledge irq */
4740 evergreen_irq_ack(rdev);
4741 evergreen_disable_interrupt_state(rdev);
4744 void evergreen_irq_suspend(struct radeon_device *rdev)
4746 evergreen_irq_disable(rdev);
4747 r600_rlc_stop(rdev);
4750 static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
4754 if (rdev->wb.enabled)
4755 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4757 wptr = RREG32(IH_RB_WPTR);
4759 if (wptr & RB_OVERFLOW) {
4760 /* When a ring buffer overflow happen start parsing interrupt
4761 * from the last not overwritten vector (wptr + 16). Hopefully
4762 * this should allow us to catchup.
4764 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
4765 wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
4766 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4767 tmp = RREG32(IH_RB_CNTL);
4768 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4769 WREG32(IH_RB_CNTL, tmp);
4770 wptr &= ~RB_OVERFLOW;
4772 return (wptr & rdev->ih.ptr_mask);
4775 int evergreen_irq_process(struct radeon_device *rdev)
4779 u32 src_id, src_data;
4781 bool queue_hotplug = false;
4782 bool queue_hdmi = false;
4783 bool queue_thermal = false;
4786 if (!rdev->ih.enabled || rdev->shutdown)
4789 wptr = evergreen_get_ih_wptr(rdev);
4792 /* is somebody else already processing irqs? */
4793 if (atomic_xchg(&rdev->ih.lock, 1))
4796 rptr = rdev->ih.rptr;
4797 DRM_DEBUG("r600_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4799 /* Order reading of wptr vs. reading of IH ring data */
4802 /* display interrupts */
4803 evergreen_irq_ack(rdev);
4805 while (rptr != wptr) {
4806 /* wptr/rptr are in bytes! */
4807 ring_index = rptr / 4;
4808 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4809 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4812 case 1: /* D1 vblank/vline */
4814 case 0: /* D1 vblank */
4815 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
4816 if (rdev->irq.crtc_vblank_int[0]) {
4817 drm_handle_vblank(rdev->ddev, 0);
4818 rdev->pm.vblank_sync = true;
4819 wake_up(&rdev->irq.vblank_queue);
4821 if (atomic_read(&rdev->irq.pflip[0]))
4822 radeon_crtc_handle_flip(rdev, 0);
4823 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
4824 DRM_DEBUG("IH: D1 vblank\n");
4827 case 1: /* D1 vline */
4828 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
4829 rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
4830 DRM_DEBUG("IH: D1 vline\n");
4834 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4838 case 2: /* D2 vblank/vline */
4840 case 0: /* D2 vblank */
4841 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
4842 if (rdev->irq.crtc_vblank_int[1]) {
4843 drm_handle_vblank(rdev->ddev, 1);
4844 rdev->pm.vblank_sync = true;
4845 wake_up(&rdev->irq.vblank_queue);
4847 if (atomic_read(&rdev->irq.pflip[1]))
4848 radeon_crtc_handle_flip(rdev, 1);
4849 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
4850 DRM_DEBUG("IH: D2 vblank\n");
4853 case 1: /* D2 vline */
4854 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
4855 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
4856 DRM_DEBUG("IH: D2 vline\n");
4860 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4864 case 3: /* D3 vblank/vline */
4866 case 0: /* D3 vblank */
4867 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
4868 if (rdev->irq.crtc_vblank_int[2]) {
4869 drm_handle_vblank(rdev->ddev, 2);
4870 rdev->pm.vblank_sync = true;
4871 wake_up(&rdev->irq.vblank_queue);
4873 if (atomic_read(&rdev->irq.pflip[2]))
4874 radeon_crtc_handle_flip(rdev, 2);
4875 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
4876 DRM_DEBUG("IH: D3 vblank\n");
4879 case 1: /* D3 vline */
4880 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
4881 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
4882 DRM_DEBUG("IH: D3 vline\n");
4886 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4890 case 4: /* D4 vblank/vline */
4892 case 0: /* D4 vblank */
4893 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
4894 if (rdev->irq.crtc_vblank_int[3]) {
4895 drm_handle_vblank(rdev->ddev, 3);
4896 rdev->pm.vblank_sync = true;
4897 wake_up(&rdev->irq.vblank_queue);
4899 if (atomic_read(&rdev->irq.pflip[3]))
4900 radeon_crtc_handle_flip(rdev, 3);
4901 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
4902 DRM_DEBUG("IH: D4 vblank\n");
4905 case 1: /* D4 vline */
4906 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
4907 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
4908 DRM_DEBUG("IH: D4 vline\n");
4912 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4916 case 5: /* D5 vblank/vline */
4918 case 0: /* D5 vblank */
4919 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
4920 if (rdev->irq.crtc_vblank_int[4]) {
4921 drm_handle_vblank(rdev->ddev, 4);
4922 rdev->pm.vblank_sync = true;
4923 wake_up(&rdev->irq.vblank_queue);
4925 if (atomic_read(&rdev->irq.pflip[4]))
4926 radeon_crtc_handle_flip(rdev, 4);
4927 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
4928 DRM_DEBUG("IH: D5 vblank\n");
4931 case 1: /* D5 vline */
4932 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
4933 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
4934 DRM_DEBUG("IH: D5 vline\n");
4938 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4942 case 6: /* D6 vblank/vline */
4944 case 0: /* D6 vblank */
4945 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
4946 if (rdev->irq.crtc_vblank_int[5]) {
4947 drm_handle_vblank(rdev->ddev, 5);
4948 rdev->pm.vblank_sync = true;
4949 wake_up(&rdev->irq.vblank_queue);
4951 if (atomic_read(&rdev->irq.pflip[5]))
4952 radeon_crtc_handle_flip(rdev, 5);
4953 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
4954 DRM_DEBUG("IH: D6 vblank\n");
4957 case 1: /* D6 vline */
4958 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
4959 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
4960 DRM_DEBUG("IH: D6 vline\n");
4964 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4968 case 8: /* D1 page flip */
4969 case 10: /* D2 page flip */
4970 case 12: /* D3 page flip */
4971 case 14: /* D4 page flip */
4972 case 16: /* D5 page flip */
4973 case 18: /* D6 page flip */
4974 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
4975 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
4977 case 42: /* HPD hotplug */
4980 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4981 rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
4982 queue_hotplug = true;
4983 DRM_DEBUG("IH: HPD1\n");
4987 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4988 rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
4989 queue_hotplug = true;
4990 DRM_DEBUG("IH: HPD2\n");
4994 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4995 rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
4996 queue_hotplug = true;
4997 DRM_DEBUG("IH: HPD3\n");
5001 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
5002 rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
5003 queue_hotplug = true;
5004 DRM_DEBUG("IH: HPD4\n");
5008 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
5009 rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
5010 queue_hotplug = true;
5011 DRM_DEBUG("IH: HPD5\n");
5015 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
5016 rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
5017 queue_hotplug = true;
5018 DRM_DEBUG("IH: HPD6\n");
5022 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5029 if (rdev->irq.stat_regs.evergreen.afmt_status1 & AFMT_AZ_FORMAT_WTRIG) {
5030 rdev->irq.stat_regs.evergreen.afmt_status1 &= ~AFMT_AZ_FORMAT_WTRIG;
5032 DRM_DEBUG("IH: HDMI0\n");
5036 if (rdev->irq.stat_regs.evergreen.afmt_status2 & AFMT_AZ_FORMAT_WTRIG) {
5037 rdev->irq.stat_regs.evergreen.afmt_status2 &= ~AFMT_AZ_FORMAT_WTRIG;
5039 DRM_DEBUG("IH: HDMI1\n");
5043 if (rdev->irq.stat_regs.evergreen.afmt_status3 & AFMT_AZ_FORMAT_WTRIG) {
5044 rdev->irq.stat_regs.evergreen.afmt_status3 &= ~AFMT_AZ_FORMAT_WTRIG;
5046 DRM_DEBUG("IH: HDMI2\n");
5050 if (rdev->irq.stat_regs.evergreen.afmt_status4 & AFMT_AZ_FORMAT_WTRIG) {
5051 rdev->irq.stat_regs.evergreen.afmt_status4 &= ~AFMT_AZ_FORMAT_WTRIG;
5053 DRM_DEBUG("IH: HDMI3\n");
5057 if (rdev->irq.stat_regs.evergreen.afmt_status5 & AFMT_AZ_FORMAT_WTRIG) {
5058 rdev->irq.stat_regs.evergreen.afmt_status5 &= ~AFMT_AZ_FORMAT_WTRIG;
5060 DRM_DEBUG("IH: HDMI4\n");
5064 if (rdev->irq.stat_regs.evergreen.afmt_status6 & AFMT_AZ_FORMAT_WTRIG) {
5065 rdev->irq.stat_regs.evergreen.afmt_status6 &= ~AFMT_AZ_FORMAT_WTRIG;
5067 DRM_DEBUG("IH: HDMI5\n");
5071 DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
5075 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
5076 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
5080 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
5081 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
5082 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
5083 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
5085 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
5087 cayman_vm_decode_fault(rdev, status, addr);
5088 /* reset addr and status */
5089 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
5091 case 176: /* CP_INT in ring buffer */
5092 case 177: /* CP_INT in IB1 */
5093 case 178: /* CP_INT in IB2 */
5094 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
5095 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5097 case 181: /* CP EOP event */
5098 DRM_DEBUG("IH: CP EOP\n");
5099 if (rdev->family >= CHIP_CAYMAN) {
5102 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5105 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5108 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5112 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
5114 case 224: /* DMA trap event */
5115 DRM_DEBUG("IH: DMA trap\n");
5116 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
5118 case 230: /* thermal low to high */
5119 DRM_DEBUG("IH: thermal low to high\n");
5120 rdev->pm.dpm.thermal.high_to_low = false;
5121 queue_thermal = true;
5123 case 231: /* thermal high to low */
5124 DRM_DEBUG("IH: thermal high to low\n");
5125 rdev->pm.dpm.thermal.high_to_low = true;
5126 queue_thermal = true;
5128 case 233: /* GUI IDLE */
5129 DRM_DEBUG("IH: GUI idle\n");
5131 case 244: /* DMA trap event */
5132 if (rdev->family >= CHIP_CAYMAN) {
5133 DRM_DEBUG("IH: DMA1 trap\n");
5134 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
5138 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
5142 /* wptr/rptr are in bytes! */
5144 rptr &= rdev->ih.ptr_mask;
5147 schedule_work(&rdev->hotplug_work);
5149 schedule_work(&rdev->audio_work);
5150 if (queue_thermal && rdev->pm.dpm_enabled)
5151 schedule_work(&rdev->pm.dpm.thermal.work);
5152 rdev->ih.rptr = rptr;
5153 WREG32(IH_RB_RPTR, rdev->ih.rptr);
5154 atomic_set(&rdev->ih.lock, 0);
5156 /* make sure wptr hasn't changed while processing */
5157 wptr = evergreen_get_ih_wptr(rdev);
5164 static int evergreen_startup(struct radeon_device *rdev)
5166 struct radeon_ring *ring;
5169 /* enable pcie gen2 link */
5170 evergreen_pcie_gen2_enable(rdev);
5172 evergreen_program_aspm(rdev);
5174 /* scratch needs to be initialized before MC */
5175 r = r600_vram_scratch_init(rdev);
5179 evergreen_mc_program(rdev);
5181 if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
5182 r = ni_mc_load_microcode(rdev);
5184 DRM_ERROR("Failed to load MC firmware!\n");
5189 if (rdev->flags & RADEON_IS_AGP) {
5190 evergreen_agp_enable(rdev);
5192 r = evergreen_pcie_gart_enable(rdev);
5196 evergreen_gpu_init(rdev);
5198 /* allocate rlc buffers */
5199 if (rdev->flags & RADEON_IS_IGP) {
5200 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5201 rdev->rlc.reg_list_size =
5202 (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
5203 rdev->rlc.cs_data = evergreen_cs_data;
5204 r = sumo_rlc_init(rdev);
5206 DRM_ERROR("Failed to init rlc BOs!\n");
5211 /* allocate wb buffer */
5212 r = radeon_wb_init(rdev);
5216 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5218 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5222 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5224 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5228 r = uvd_v2_2_resume(rdev);
5230 r = radeon_fence_driver_start_ring(rdev,
5231 R600_RING_TYPE_UVD_INDEX);
5233 dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5237 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
5240 if (!rdev->irq.installed) {
5241 r = radeon_irq_kms_init(rdev);
5246 r = r600_irq_init(rdev);
5248 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5249 radeon_irq_kms_fini(rdev);
5252 evergreen_irq_set(rdev);
5254 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5255 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5260 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5261 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5262 DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5266 r = evergreen_cp_load_microcode(rdev);
5269 r = evergreen_cp_resume(rdev);
5272 r = r600_dma_resume(rdev);
5276 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5277 if (ring->ring_size) {
5278 r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
5281 r = uvd_v1_0_init(rdev);
5284 DRM_ERROR("radeon: error initializing UVD (%d).\n", r);
5287 r = radeon_ib_pool_init(rdev);
5289 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5293 r = r600_audio_init(rdev);
5295 DRM_ERROR("radeon: audio init failed\n");
5302 int evergreen_resume(struct radeon_device *rdev)
5306 /* reset the asic, the gfx blocks are often in a bad state
5307 * after the driver is unloaded or after a resume
5309 if (radeon_asic_reset(rdev))
5310 dev_warn(rdev->dev, "GPU reset failed !\n");
5311 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5312 * posting will perform necessary task to bring back GPU into good
5316 atom_asic_init(rdev->mode_info.atom_context);
5318 /* init golden registers */
5319 evergreen_init_golden_registers(rdev);
5321 if (rdev->pm.pm_method == PM_METHOD_DPM)
5322 radeon_pm_resume(rdev);
5324 rdev->accel_working = true;
5325 r = evergreen_startup(rdev);
5327 DRM_ERROR("evergreen startup failed on resume\n");
5328 rdev->accel_working = false;
5336 int evergreen_suspend(struct radeon_device *rdev)
5338 radeon_pm_suspend(rdev);
5339 r600_audio_fini(rdev);
5340 uvd_v1_0_fini(rdev);
5341 radeon_uvd_suspend(rdev);
5343 r600_dma_stop(rdev);
5344 evergreen_irq_suspend(rdev);
5345 radeon_wb_disable(rdev);
5346 evergreen_pcie_gart_disable(rdev);
5351 /* Plan is to move initialization in that function and use
5352 * helper function so that radeon_device_init pretty much
5353 * do nothing more than calling asic specific function. This
5354 * should also allow to remove a bunch of callback function
5357 int evergreen_init(struct radeon_device *rdev)
5362 if (!radeon_get_bios(rdev)) {
5363 if (ASIC_IS_AVIVO(rdev))
5366 /* Must be an ATOMBIOS */
5367 if (!rdev->is_atom_bios) {
5368 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
5371 r = radeon_atombios_init(rdev);
5374 /* reset the asic, the gfx blocks are often in a bad state
5375 * after the driver is unloaded or after a resume
5377 if (radeon_asic_reset(rdev))
5378 dev_warn(rdev->dev, "GPU reset failed !\n");
5379 /* Post card if necessary */
5380 if (!radeon_card_posted(rdev)) {
5382 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5385 DRM_INFO("GPU not posted. posting now...\n");
5386 atom_asic_init(rdev->mode_info.atom_context);
5388 /* init golden registers */
5389 evergreen_init_golden_registers(rdev);
5390 /* Initialize scratch registers */
5391 r600_scratch_init(rdev);
5392 /* Initialize surface registers */
5393 radeon_surface_init(rdev);
5394 /* Initialize clocks */
5395 radeon_get_clock_info(rdev->ddev);
5397 r = radeon_fence_driver_init(rdev);
5400 /* initialize AGP */
5401 if (rdev->flags & RADEON_IS_AGP) {
5402 r = radeon_agp_init(rdev);
5404 radeon_agp_disable(rdev);
5406 /* initialize memory controller */
5407 r = evergreen_mc_init(rdev);
5410 /* Memory manager */
5411 r = radeon_bo_init(rdev);
5415 if (ASIC_IS_DCE5(rdev)) {
5416 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5417 r = ni_init_microcode(rdev);
5419 DRM_ERROR("Failed to load firmware!\n");
5424 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5425 r = r600_init_microcode(rdev);
5427 DRM_ERROR("Failed to load firmware!\n");
5433 /* Initialize power management */
5434 radeon_pm_init(rdev);
5436 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5437 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
5439 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5440 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5442 r = radeon_uvd_init(rdev);
5444 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
5445 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX],
5449 rdev->ih.ring_obj = NULL;
5450 r600_ih_ring_init(rdev, 64 * 1024);
5452 r = r600_pcie_gart_init(rdev);
5456 rdev->accel_working = true;
5457 r = evergreen_startup(rdev);
5459 dev_err(rdev->dev, "disabling GPU acceleration\n");
5461 r600_dma_fini(rdev);
5462 r600_irq_fini(rdev);
5463 if (rdev->flags & RADEON_IS_IGP)
5464 sumo_rlc_fini(rdev);
5465 radeon_wb_fini(rdev);
5466 radeon_ib_pool_fini(rdev);
5467 radeon_irq_kms_fini(rdev);
5468 evergreen_pcie_gart_fini(rdev);
5469 rdev->accel_working = false;
5472 /* Don't start up if the MC ucode is missing on BTC parts.
5473 * The default clocks and voltages before the MC ucode
5474 * is loaded are not suffient for advanced operations.
5476 if (ASIC_IS_DCE5(rdev)) {
5477 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5478 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5486 void evergreen_fini(struct radeon_device *rdev)
5488 radeon_pm_fini(rdev);
5489 r600_audio_fini(rdev);
5491 r600_dma_fini(rdev);
5492 r600_irq_fini(rdev);
5493 if (rdev->flags & RADEON_IS_IGP)
5494 sumo_rlc_fini(rdev);
5495 radeon_wb_fini(rdev);
5496 radeon_ib_pool_fini(rdev);
5497 radeon_irq_kms_fini(rdev);
5498 uvd_v1_0_fini(rdev);
5499 radeon_uvd_fini(rdev);
5500 evergreen_pcie_gart_fini(rdev);
5501 r600_vram_scratch_fini(rdev);
5502 radeon_gem_fini(rdev);
5503 radeon_fence_driver_fini(rdev);
5504 radeon_agp_fini(rdev);
5505 radeon_bo_fini(rdev);
5506 radeon_atombios_fini(rdev);
5511 void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
5513 u32 link_width_cntl, speed_cntl;
5515 if (radeon_pcie_gen2 == 0)
5518 if (rdev->flags & RADEON_IS_IGP)
5521 if (!(rdev->flags & RADEON_IS_PCIE))
5524 /* x2 cards have a special sequence */
5525 if (ASIC_IS_X2(rdev))
5528 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5529 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
5532 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5533 if (speed_cntl & LC_CURRENT_DATA_RATE) {
5534 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5538 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5540 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5541 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5543 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5544 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5545 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5547 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5548 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
5549 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5551 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5552 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
5553 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5555 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5556 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
5557 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5559 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5560 speed_cntl |= LC_GEN2_EN_STRAP;
5561 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5564 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5565 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5567 link_width_cntl |= LC_UPCONFIGURE_DIS;
5569 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5570 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5574 void evergreen_program_aspm(struct radeon_device *rdev)
5577 u32 pcie_lc_cntl, pcie_lc_cntl_old;
5578 bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5579 /* fusion_platform = true
5580 * if the system is a fusion system
5581 * (APU or DGPU in a fusion system).
5582 * todo: check if the system is a fusion platform.
5584 bool fusion_platform = false;
5586 if (radeon_aspm == 0)
5589 if (!(rdev->flags & RADEON_IS_PCIE))
5592 switch (rdev->family) {
5605 disable_l0s = false;
5609 if (rdev->flags & RADEON_IS_IGP)
5610 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5612 data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5613 if (fusion_platform)
5618 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5620 data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5621 if (fusion_platform)
5626 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5628 pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5629 pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5631 if (rdev->family >= CHIP_BARTS)
5632 pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5634 pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5638 if (rdev->family >= CHIP_BARTS)
5639 pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5641 pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5643 if (!disable_plloff_in_l1) {
5644 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5645 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5646 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5648 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5650 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5651 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5652 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5654 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5656 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5657 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5658 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5660 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5662 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5663 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5664 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5666 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5668 if (rdev->family >= CHIP_BARTS) {
5669 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5670 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5671 data |= PLL_RAMP_UP_TIME_0(4);
5673 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5675 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5676 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5677 data |= PLL_RAMP_UP_TIME_1(4);
5679 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5681 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5682 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5683 data |= PLL_RAMP_UP_TIME_0(4);
5685 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5687 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5688 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5689 data |= PLL_RAMP_UP_TIME_1(4);
5691 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5694 data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5695 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5696 data |= LC_DYN_LANES_PWR_STATE(3);
5698 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5700 if (rdev->family >= CHIP_BARTS) {
5701 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5702 data &= ~LS2_EXIT_TIME_MASK;
5703 data |= LS2_EXIT_TIME(1);
5705 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5707 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5708 data &= ~LS2_EXIT_TIME_MASK;
5709 data |= LS2_EXIT_TIME(1);
5711 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5716 /* evergreen parts only */
5717 if (rdev->family < CHIP_BARTS)
5718 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5720 if (pcie_lc_cntl != pcie_lc_cntl_old)
5721 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);