2 * Copyright 2011 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
22 * Authors: Alex Deucher
25 #include <linux/firmware.h>
26 #include <linux/module.h>
27 #include <linux/pci.h>
28 #include <linux/slab.h>
30 #include <drm/drm_vblank.h>
31 #include <drm/radeon_drm.h>
34 #include "clearstate_si.h"
35 #include "evergreen.h"
38 #include "radeon_asic.h"
39 #include "radeon_audio.h"
40 #include "radeon_ucode.h"
41 #include "si_blit_shaders.h"
46 MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
47 MODULE_FIRMWARE("radeon/TAHITI_me.bin");
48 MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
49 MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
50 MODULE_FIRMWARE("radeon/TAHITI_mc2.bin");
51 MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
52 MODULE_FIRMWARE("radeon/TAHITI_smc.bin");
54 MODULE_FIRMWARE("radeon/tahiti_pfp.bin");
55 MODULE_FIRMWARE("radeon/tahiti_me.bin");
56 MODULE_FIRMWARE("radeon/tahiti_ce.bin");
57 MODULE_FIRMWARE("radeon/tahiti_mc.bin");
58 MODULE_FIRMWARE("radeon/tahiti_rlc.bin");
59 MODULE_FIRMWARE("radeon/tahiti_smc.bin");
61 MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
62 MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
63 MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
64 MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
65 MODULE_FIRMWARE("radeon/PITCAIRN_mc2.bin");
66 MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
67 MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin");
69 MODULE_FIRMWARE("radeon/pitcairn_pfp.bin");
70 MODULE_FIRMWARE("radeon/pitcairn_me.bin");
71 MODULE_FIRMWARE("radeon/pitcairn_ce.bin");
72 MODULE_FIRMWARE("radeon/pitcairn_mc.bin");
73 MODULE_FIRMWARE("radeon/pitcairn_rlc.bin");
74 MODULE_FIRMWARE("radeon/pitcairn_smc.bin");
75 MODULE_FIRMWARE("radeon/pitcairn_k_smc.bin");
77 MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
78 MODULE_FIRMWARE("radeon/VERDE_me.bin");
79 MODULE_FIRMWARE("radeon/VERDE_ce.bin");
80 MODULE_FIRMWARE("radeon/VERDE_mc.bin");
81 MODULE_FIRMWARE("radeon/VERDE_mc2.bin");
82 MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
83 MODULE_FIRMWARE("radeon/VERDE_smc.bin");
85 MODULE_FIRMWARE("radeon/verde_pfp.bin");
86 MODULE_FIRMWARE("radeon/verde_me.bin");
87 MODULE_FIRMWARE("radeon/verde_ce.bin");
88 MODULE_FIRMWARE("radeon/verde_mc.bin");
89 MODULE_FIRMWARE("radeon/verde_rlc.bin");
90 MODULE_FIRMWARE("radeon/verde_smc.bin");
91 MODULE_FIRMWARE("radeon/verde_k_smc.bin");
93 MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
94 MODULE_FIRMWARE("radeon/OLAND_me.bin");
95 MODULE_FIRMWARE("radeon/OLAND_ce.bin");
96 MODULE_FIRMWARE("radeon/OLAND_mc.bin");
97 MODULE_FIRMWARE("radeon/OLAND_mc2.bin");
98 MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
99 MODULE_FIRMWARE("radeon/OLAND_smc.bin");
101 MODULE_FIRMWARE("radeon/oland_pfp.bin");
102 MODULE_FIRMWARE("radeon/oland_me.bin");
103 MODULE_FIRMWARE("radeon/oland_ce.bin");
104 MODULE_FIRMWARE("radeon/oland_mc.bin");
105 MODULE_FIRMWARE("radeon/oland_rlc.bin");
106 MODULE_FIRMWARE("radeon/oland_smc.bin");
107 MODULE_FIRMWARE("radeon/oland_k_smc.bin");
109 MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
110 MODULE_FIRMWARE("radeon/HAINAN_me.bin");
111 MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
112 MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
113 MODULE_FIRMWARE("radeon/HAINAN_mc2.bin");
114 MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
115 MODULE_FIRMWARE("radeon/HAINAN_smc.bin");
117 MODULE_FIRMWARE("radeon/hainan_pfp.bin");
118 MODULE_FIRMWARE("radeon/hainan_me.bin");
119 MODULE_FIRMWARE("radeon/hainan_ce.bin");
120 MODULE_FIRMWARE("radeon/hainan_mc.bin");
121 MODULE_FIRMWARE("radeon/hainan_rlc.bin");
122 MODULE_FIRMWARE("radeon/hainan_smc.bin");
123 MODULE_FIRMWARE("radeon/hainan_k_smc.bin");
124 MODULE_FIRMWARE("radeon/banks_k_2_smc.bin");
126 MODULE_FIRMWARE("radeon/si58_mc.bin");
128 static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
129 static void si_pcie_gen3_enable(struct radeon_device *rdev);
130 static void si_program_aspm(struct radeon_device *rdev);
131 extern void sumo_rlc_fini(struct radeon_device *rdev);
132 extern int sumo_rlc_init(struct radeon_device *rdev);
133 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
135 static void si_init_pg(struct radeon_device *rdev);
136 static void si_init_cg(struct radeon_device *rdev);
137 static void si_fini_pg(struct radeon_device *rdev);
138 static void si_fini_cg(struct radeon_device *rdev);
139 static void si_rlc_stop(struct radeon_device *rdev);
141 static const u32 crtc_offsets[] = {
142 EVERGREEN_CRTC0_REGISTER_OFFSET,
143 EVERGREEN_CRTC1_REGISTER_OFFSET,
144 EVERGREEN_CRTC2_REGISTER_OFFSET,
145 EVERGREEN_CRTC3_REGISTER_OFFSET,
146 EVERGREEN_CRTC4_REGISTER_OFFSET,
147 EVERGREEN_CRTC5_REGISTER_OFFSET
150 static const u32 si_disp_int_status[] = {
151 DISP_INTERRUPT_STATUS,
152 DISP_INTERRUPT_STATUS_CONTINUE,
153 DISP_INTERRUPT_STATUS_CONTINUE2,
154 DISP_INTERRUPT_STATUS_CONTINUE3,
155 DISP_INTERRUPT_STATUS_CONTINUE4,
156 DISP_INTERRUPT_STATUS_CONTINUE5
159 #define DC_HPDx_CONTROL(x) (DC_HPD1_CONTROL + (x * 0xc))
160 #define DC_HPDx_INT_CONTROL(x) (DC_HPD1_INT_CONTROL + (x * 0xc))
161 #define DC_HPDx_INT_STATUS_REG(x) (DC_HPD1_INT_STATUS + (x * 0xc))
163 static const u32 verde_rlc_save_restore_register_list[] = {
164 (0x8000 << 16) | (0x98f4 >> 2),
166 (0x8040 << 16) | (0x98f4 >> 2),
168 (0x8000 << 16) | (0xe80 >> 2),
170 (0x8040 << 16) | (0xe80 >> 2),
172 (0x8000 << 16) | (0x89bc >> 2),
174 (0x8040 << 16) | (0x89bc >> 2),
176 (0x8000 << 16) | (0x8c1c >> 2),
178 (0x8040 << 16) | (0x8c1c >> 2),
180 (0x9c00 << 16) | (0x98f0 >> 2),
182 (0x9c00 << 16) | (0xe7c >> 2),
184 (0x8000 << 16) | (0x9148 >> 2),
186 (0x8040 << 16) | (0x9148 >> 2),
188 (0x9c00 << 16) | (0x9150 >> 2),
190 (0x9c00 << 16) | (0x897c >> 2),
192 (0x9c00 << 16) | (0x8d8c >> 2),
194 (0x9c00 << 16) | (0xac54 >> 2),
197 (0x9c00 << 16) | (0x98f8 >> 2),
199 (0x9c00 << 16) | (0x9910 >> 2),
201 (0x9c00 << 16) | (0x9914 >> 2),
203 (0x9c00 << 16) | (0x9918 >> 2),
205 (0x9c00 << 16) | (0x991c >> 2),
207 (0x9c00 << 16) | (0x9920 >> 2),
209 (0x9c00 << 16) | (0x9924 >> 2),
211 (0x9c00 << 16) | (0x9928 >> 2),
213 (0x9c00 << 16) | (0x992c >> 2),
215 (0x9c00 << 16) | (0x9930 >> 2),
217 (0x9c00 << 16) | (0x9934 >> 2),
219 (0x9c00 << 16) | (0x9938 >> 2),
221 (0x9c00 << 16) | (0x993c >> 2),
223 (0x9c00 << 16) | (0x9940 >> 2),
225 (0x9c00 << 16) | (0x9944 >> 2),
227 (0x9c00 << 16) | (0x9948 >> 2),
229 (0x9c00 << 16) | (0x994c >> 2),
231 (0x9c00 << 16) | (0x9950 >> 2),
233 (0x9c00 << 16) | (0x9954 >> 2),
235 (0x9c00 << 16) | (0x9958 >> 2),
237 (0x9c00 << 16) | (0x995c >> 2),
239 (0x9c00 << 16) | (0x9960 >> 2),
241 (0x9c00 << 16) | (0x9964 >> 2),
243 (0x9c00 << 16) | (0x9968 >> 2),
245 (0x9c00 << 16) | (0x996c >> 2),
247 (0x9c00 << 16) | (0x9970 >> 2),
249 (0x9c00 << 16) | (0x9974 >> 2),
251 (0x9c00 << 16) | (0x9978 >> 2),
253 (0x9c00 << 16) | (0x997c >> 2),
255 (0x9c00 << 16) | (0x9980 >> 2),
257 (0x9c00 << 16) | (0x9984 >> 2),
259 (0x9c00 << 16) | (0x9988 >> 2),
261 (0x9c00 << 16) | (0x998c >> 2),
263 (0x9c00 << 16) | (0x8c00 >> 2),
265 (0x9c00 << 16) | (0x8c14 >> 2),
267 (0x9c00 << 16) | (0x8c04 >> 2),
269 (0x9c00 << 16) | (0x8c08 >> 2),
271 (0x8000 << 16) | (0x9b7c >> 2),
273 (0x8040 << 16) | (0x9b7c >> 2),
275 (0x8000 << 16) | (0xe84 >> 2),
277 (0x8040 << 16) | (0xe84 >> 2),
279 (0x8000 << 16) | (0x89c0 >> 2),
281 (0x8040 << 16) | (0x89c0 >> 2),
283 (0x8000 << 16) | (0x914c >> 2),
285 (0x8040 << 16) | (0x914c >> 2),
287 (0x8000 << 16) | (0x8c20 >> 2),
289 (0x8040 << 16) | (0x8c20 >> 2),
291 (0x8000 << 16) | (0x9354 >> 2),
293 (0x8040 << 16) | (0x9354 >> 2),
295 (0x9c00 << 16) | (0x9060 >> 2),
297 (0x9c00 << 16) | (0x9364 >> 2),
299 (0x9c00 << 16) | (0x9100 >> 2),
301 (0x9c00 << 16) | (0x913c >> 2),
303 (0x8000 << 16) | (0x90e0 >> 2),
305 (0x8000 << 16) | (0x90e4 >> 2),
307 (0x8000 << 16) | (0x90e8 >> 2),
309 (0x8040 << 16) | (0x90e0 >> 2),
311 (0x8040 << 16) | (0x90e4 >> 2),
313 (0x8040 << 16) | (0x90e8 >> 2),
315 (0x9c00 << 16) | (0x8bcc >> 2),
317 (0x9c00 << 16) | (0x8b24 >> 2),
319 (0x9c00 << 16) | (0x88c4 >> 2),
321 (0x9c00 << 16) | (0x8e50 >> 2),
323 (0x9c00 << 16) | (0x8c0c >> 2),
325 (0x9c00 << 16) | (0x8e58 >> 2),
327 (0x9c00 << 16) | (0x8e5c >> 2),
329 (0x9c00 << 16) | (0x9508 >> 2),
331 (0x9c00 << 16) | (0x950c >> 2),
333 (0x9c00 << 16) | (0x9494 >> 2),
335 (0x9c00 << 16) | (0xac0c >> 2),
337 (0x9c00 << 16) | (0xac10 >> 2),
339 (0x9c00 << 16) | (0xac14 >> 2),
341 (0x9c00 << 16) | (0xae00 >> 2),
343 (0x9c00 << 16) | (0xac08 >> 2),
345 (0x9c00 << 16) | (0x88d4 >> 2),
347 (0x9c00 << 16) | (0x88c8 >> 2),
349 (0x9c00 << 16) | (0x88cc >> 2),
351 (0x9c00 << 16) | (0x89b0 >> 2),
353 (0x9c00 << 16) | (0x8b10 >> 2),
355 (0x9c00 << 16) | (0x8a14 >> 2),
357 (0x9c00 << 16) | (0x9830 >> 2),
359 (0x9c00 << 16) | (0x9834 >> 2),
361 (0x9c00 << 16) | (0x9838 >> 2),
363 (0x9c00 << 16) | (0x9a10 >> 2),
365 (0x8000 << 16) | (0x9870 >> 2),
367 (0x8000 << 16) | (0x9874 >> 2),
369 (0x8001 << 16) | (0x9870 >> 2),
371 (0x8001 << 16) | (0x9874 >> 2),
373 (0x8040 << 16) | (0x9870 >> 2),
375 (0x8040 << 16) | (0x9874 >> 2),
377 (0x8041 << 16) | (0x9870 >> 2),
379 (0x8041 << 16) | (0x9874 >> 2),
384 static const u32 tahiti_golden_rlc_registers[] = {
385 0xc424, 0xffffffff, 0x00601005,
386 0xc47c, 0xffffffff, 0x10104040,
387 0xc488, 0xffffffff, 0x0100000a,
388 0xc314, 0xffffffff, 0x00000800,
389 0xc30c, 0xffffffff, 0x800000f4,
390 0xf4a8, 0xffffffff, 0x00000000
393 static const u32 tahiti_golden_registers[] = {
394 0x9a10, 0x00010000, 0x00018208,
395 0x9830, 0xffffffff, 0x00000000,
396 0x9834, 0xf00fffff, 0x00000400,
397 0x9838, 0x0002021c, 0x00020200,
398 0xc78, 0x00000080, 0x00000000,
399 0xd030, 0x000300c0, 0x00800040,
400 0xd830, 0x000300c0, 0x00800040,
401 0x5bb0, 0x000000f0, 0x00000070,
402 0x5bc0, 0x00200000, 0x50100000,
403 0x7030, 0x31000311, 0x00000011,
404 0x277c, 0x00000003, 0x000007ff,
405 0x240c, 0x000007ff, 0x00000000,
406 0x8a14, 0xf000001f, 0x00000007,
407 0x8b24, 0xffffffff, 0x00ffffff,
408 0x8b10, 0x0000ff0f, 0x00000000,
409 0x28a4c, 0x07ffffff, 0x4e000000,
410 0x28350, 0x3f3f3fff, 0x2a00126a,
411 0x30, 0x000000ff, 0x0040,
412 0x34, 0x00000040, 0x00004040,
413 0x9100, 0x07ffffff, 0x03000000,
414 0x8e88, 0x01ff1f3f, 0x00000000,
415 0x8e84, 0x01ff1f3f, 0x00000000,
416 0x9060, 0x0000007f, 0x00000020,
417 0x9508, 0x00010000, 0x00010000,
418 0xac14, 0x00000200, 0x000002fb,
419 0xac10, 0xffffffff, 0x0000543b,
420 0xac0c, 0xffffffff, 0xa9210876,
421 0x88d0, 0xffffffff, 0x000fff40,
422 0x88d4, 0x0000001f, 0x00000010,
423 0x1410, 0x20000000, 0x20fffed8,
424 0x15c0, 0x000c0fc0, 0x000c0400
427 static const u32 tahiti_golden_registers2[] = {
428 0xc64, 0x00000001, 0x00000001
431 static const u32 pitcairn_golden_rlc_registers[] = {
432 0xc424, 0xffffffff, 0x00601004,
433 0xc47c, 0xffffffff, 0x10102020,
434 0xc488, 0xffffffff, 0x01000020,
435 0xc314, 0xffffffff, 0x00000800,
436 0xc30c, 0xffffffff, 0x800000a4
439 static const u32 pitcairn_golden_registers[] = {
440 0x9a10, 0x00010000, 0x00018208,
441 0x9830, 0xffffffff, 0x00000000,
442 0x9834, 0xf00fffff, 0x00000400,
443 0x9838, 0x0002021c, 0x00020200,
444 0xc78, 0x00000080, 0x00000000,
445 0xd030, 0x000300c0, 0x00800040,
446 0xd830, 0x000300c0, 0x00800040,
447 0x5bb0, 0x000000f0, 0x00000070,
448 0x5bc0, 0x00200000, 0x50100000,
449 0x7030, 0x31000311, 0x00000011,
450 0x2ae4, 0x00073ffe, 0x000022a2,
451 0x240c, 0x000007ff, 0x00000000,
452 0x8a14, 0xf000001f, 0x00000007,
453 0x8b24, 0xffffffff, 0x00ffffff,
454 0x8b10, 0x0000ff0f, 0x00000000,
455 0x28a4c, 0x07ffffff, 0x4e000000,
456 0x28350, 0x3f3f3fff, 0x2a00126a,
457 0x30, 0x000000ff, 0x0040,
458 0x34, 0x00000040, 0x00004040,
459 0x9100, 0x07ffffff, 0x03000000,
460 0x9060, 0x0000007f, 0x00000020,
461 0x9508, 0x00010000, 0x00010000,
462 0xac14, 0x000003ff, 0x000000f7,
463 0xac10, 0xffffffff, 0x00000000,
464 0xac0c, 0xffffffff, 0x32761054,
465 0x88d4, 0x0000001f, 0x00000010,
466 0x15c0, 0x000c0fc0, 0x000c0400
469 static const u32 verde_golden_rlc_registers[] = {
470 0xc424, 0xffffffff, 0x033f1005,
471 0xc47c, 0xffffffff, 0x10808020,
472 0xc488, 0xffffffff, 0x00800008,
473 0xc314, 0xffffffff, 0x00001000,
474 0xc30c, 0xffffffff, 0x80010014
477 static const u32 verde_golden_registers[] = {
478 0x9a10, 0x00010000, 0x00018208,
479 0x9830, 0xffffffff, 0x00000000,
480 0x9834, 0xf00fffff, 0x00000400,
481 0x9838, 0x0002021c, 0x00020200,
482 0xc78, 0x00000080, 0x00000000,
483 0xd030, 0x000300c0, 0x00800040,
484 0xd030, 0x000300c0, 0x00800040,
485 0xd830, 0x000300c0, 0x00800040,
486 0xd830, 0x000300c0, 0x00800040,
487 0x5bb0, 0x000000f0, 0x00000070,
488 0x5bc0, 0x00200000, 0x50100000,
489 0x7030, 0x31000311, 0x00000011,
490 0x2ae4, 0x00073ffe, 0x000022a2,
491 0x2ae4, 0x00073ffe, 0x000022a2,
492 0x2ae4, 0x00073ffe, 0x000022a2,
493 0x240c, 0x000007ff, 0x00000000,
494 0x240c, 0x000007ff, 0x00000000,
495 0x240c, 0x000007ff, 0x00000000,
496 0x8a14, 0xf000001f, 0x00000007,
497 0x8a14, 0xf000001f, 0x00000007,
498 0x8a14, 0xf000001f, 0x00000007,
499 0x8b24, 0xffffffff, 0x00ffffff,
500 0x8b10, 0x0000ff0f, 0x00000000,
501 0x28a4c, 0x07ffffff, 0x4e000000,
502 0x28350, 0x3f3f3fff, 0x0000124a,
503 0x28350, 0x3f3f3fff, 0x0000124a,
504 0x28350, 0x3f3f3fff, 0x0000124a,
505 0x30, 0x000000ff, 0x0040,
506 0x34, 0x00000040, 0x00004040,
507 0x9100, 0x07ffffff, 0x03000000,
508 0x9100, 0x07ffffff, 0x03000000,
509 0x8e88, 0x01ff1f3f, 0x00000000,
510 0x8e88, 0x01ff1f3f, 0x00000000,
511 0x8e88, 0x01ff1f3f, 0x00000000,
512 0x8e84, 0x01ff1f3f, 0x00000000,
513 0x8e84, 0x01ff1f3f, 0x00000000,
514 0x8e84, 0x01ff1f3f, 0x00000000,
515 0x9060, 0x0000007f, 0x00000020,
516 0x9508, 0x00010000, 0x00010000,
517 0xac14, 0x000003ff, 0x00000003,
518 0xac14, 0x000003ff, 0x00000003,
519 0xac14, 0x000003ff, 0x00000003,
520 0xac10, 0xffffffff, 0x00000000,
521 0xac10, 0xffffffff, 0x00000000,
522 0xac10, 0xffffffff, 0x00000000,
523 0xac0c, 0xffffffff, 0x00001032,
524 0xac0c, 0xffffffff, 0x00001032,
525 0xac0c, 0xffffffff, 0x00001032,
526 0x88d4, 0x0000001f, 0x00000010,
527 0x88d4, 0x0000001f, 0x00000010,
528 0x88d4, 0x0000001f, 0x00000010,
529 0x15c0, 0x000c0fc0, 0x000c0400
532 static const u32 oland_golden_rlc_registers[] = {
533 0xc424, 0xffffffff, 0x00601005,
534 0xc47c, 0xffffffff, 0x10104040,
535 0xc488, 0xffffffff, 0x0100000a,
536 0xc314, 0xffffffff, 0x00000800,
537 0xc30c, 0xffffffff, 0x800000f4
540 static const u32 oland_golden_registers[] = {
541 0x9a10, 0x00010000, 0x00018208,
542 0x9830, 0xffffffff, 0x00000000,
543 0x9834, 0xf00fffff, 0x00000400,
544 0x9838, 0x0002021c, 0x00020200,
545 0xc78, 0x00000080, 0x00000000,
546 0xd030, 0x000300c0, 0x00800040,
547 0xd830, 0x000300c0, 0x00800040,
548 0x5bb0, 0x000000f0, 0x00000070,
549 0x5bc0, 0x00200000, 0x50100000,
550 0x7030, 0x31000311, 0x00000011,
551 0x2ae4, 0x00073ffe, 0x000022a2,
552 0x240c, 0x000007ff, 0x00000000,
553 0x8a14, 0xf000001f, 0x00000007,
554 0x8b24, 0xffffffff, 0x00ffffff,
555 0x8b10, 0x0000ff0f, 0x00000000,
556 0x28a4c, 0x07ffffff, 0x4e000000,
557 0x28350, 0x3f3f3fff, 0x00000082,
558 0x30, 0x000000ff, 0x0040,
559 0x34, 0x00000040, 0x00004040,
560 0x9100, 0x07ffffff, 0x03000000,
561 0x9060, 0x0000007f, 0x00000020,
562 0x9508, 0x00010000, 0x00010000,
563 0xac14, 0x000003ff, 0x000000f3,
564 0xac10, 0xffffffff, 0x00000000,
565 0xac0c, 0xffffffff, 0x00003210,
566 0x88d4, 0x0000001f, 0x00000010,
567 0x15c0, 0x000c0fc0, 0x000c0400
570 static const u32 hainan_golden_registers[] = {
571 0x9a10, 0x00010000, 0x00018208,
572 0x9830, 0xffffffff, 0x00000000,
573 0x9834, 0xf00fffff, 0x00000400,
574 0x9838, 0x0002021c, 0x00020200,
575 0xd0c0, 0xff000fff, 0x00000100,
576 0xd030, 0x000300c0, 0x00800040,
577 0xd8c0, 0xff000fff, 0x00000100,
578 0xd830, 0x000300c0, 0x00800040,
579 0x2ae4, 0x00073ffe, 0x000022a2,
580 0x240c, 0x000007ff, 0x00000000,
581 0x8a14, 0xf000001f, 0x00000007,
582 0x8b24, 0xffffffff, 0x00ffffff,
583 0x8b10, 0x0000ff0f, 0x00000000,
584 0x28a4c, 0x07ffffff, 0x4e000000,
585 0x28350, 0x3f3f3fff, 0x00000000,
586 0x30, 0x000000ff, 0x0040,
587 0x34, 0x00000040, 0x00004040,
588 0x9100, 0x03e00000, 0x03600000,
589 0x9060, 0x0000007f, 0x00000020,
590 0x9508, 0x00010000, 0x00010000,
591 0xac14, 0x000003ff, 0x000000f1,
592 0xac10, 0xffffffff, 0x00000000,
593 0xac0c, 0xffffffff, 0x00003210,
594 0x88d4, 0x0000001f, 0x00000010,
595 0x15c0, 0x000c0fc0, 0x000c0400
598 static const u32 hainan_golden_registers2[] = {
599 0x98f8, 0xffffffff, 0x02010001
602 static const u32 tahiti_mgcg_cgcg_init[] = {
603 0xc400, 0xffffffff, 0xfffffffc,
604 0x802c, 0xffffffff, 0xe0000000,
605 0x9a60, 0xffffffff, 0x00000100,
606 0x92a4, 0xffffffff, 0x00000100,
607 0xc164, 0xffffffff, 0x00000100,
608 0x9774, 0xffffffff, 0x00000100,
609 0x8984, 0xffffffff, 0x06000100,
610 0x8a18, 0xffffffff, 0x00000100,
611 0x92a0, 0xffffffff, 0x00000100,
612 0xc380, 0xffffffff, 0x00000100,
613 0x8b28, 0xffffffff, 0x00000100,
614 0x9144, 0xffffffff, 0x00000100,
615 0x8d88, 0xffffffff, 0x00000100,
616 0x8d8c, 0xffffffff, 0x00000100,
617 0x9030, 0xffffffff, 0x00000100,
618 0x9034, 0xffffffff, 0x00000100,
619 0x9038, 0xffffffff, 0x00000100,
620 0x903c, 0xffffffff, 0x00000100,
621 0xad80, 0xffffffff, 0x00000100,
622 0xac54, 0xffffffff, 0x00000100,
623 0x897c, 0xffffffff, 0x06000100,
624 0x9868, 0xffffffff, 0x00000100,
625 0x9510, 0xffffffff, 0x00000100,
626 0xaf04, 0xffffffff, 0x00000100,
627 0xae04, 0xffffffff, 0x00000100,
628 0x949c, 0xffffffff, 0x00000100,
629 0x802c, 0xffffffff, 0xe0000000,
630 0x9160, 0xffffffff, 0x00010000,
631 0x9164, 0xffffffff, 0x00030002,
632 0x9168, 0xffffffff, 0x00040007,
633 0x916c, 0xffffffff, 0x00060005,
634 0x9170, 0xffffffff, 0x00090008,
635 0x9174, 0xffffffff, 0x00020001,
636 0x9178, 0xffffffff, 0x00040003,
637 0x917c, 0xffffffff, 0x00000007,
638 0x9180, 0xffffffff, 0x00060005,
639 0x9184, 0xffffffff, 0x00090008,
640 0x9188, 0xffffffff, 0x00030002,
641 0x918c, 0xffffffff, 0x00050004,
642 0x9190, 0xffffffff, 0x00000008,
643 0x9194, 0xffffffff, 0x00070006,
644 0x9198, 0xffffffff, 0x000a0009,
645 0x919c, 0xffffffff, 0x00040003,
646 0x91a0, 0xffffffff, 0x00060005,
647 0x91a4, 0xffffffff, 0x00000009,
648 0x91a8, 0xffffffff, 0x00080007,
649 0x91ac, 0xffffffff, 0x000b000a,
650 0x91b0, 0xffffffff, 0x00050004,
651 0x91b4, 0xffffffff, 0x00070006,
652 0x91b8, 0xffffffff, 0x0008000b,
653 0x91bc, 0xffffffff, 0x000a0009,
654 0x91c0, 0xffffffff, 0x000d000c,
655 0x91c4, 0xffffffff, 0x00060005,
656 0x91c8, 0xffffffff, 0x00080007,
657 0x91cc, 0xffffffff, 0x0000000b,
658 0x91d0, 0xffffffff, 0x000a0009,
659 0x91d4, 0xffffffff, 0x000d000c,
660 0x91d8, 0xffffffff, 0x00070006,
661 0x91dc, 0xffffffff, 0x00090008,
662 0x91e0, 0xffffffff, 0x0000000c,
663 0x91e4, 0xffffffff, 0x000b000a,
664 0x91e8, 0xffffffff, 0x000e000d,
665 0x91ec, 0xffffffff, 0x00080007,
666 0x91f0, 0xffffffff, 0x000a0009,
667 0x91f4, 0xffffffff, 0x0000000d,
668 0x91f8, 0xffffffff, 0x000c000b,
669 0x91fc, 0xffffffff, 0x000f000e,
670 0x9200, 0xffffffff, 0x00090008,
671 0x9204, 0xffffffff, 0x000b000a,
672 0x9208, 0xffffffff, 0x000c000f,
673 0x920c, 0xffffffff, 0x000e000d,
674 0x9210, 0xffffffff, 0x00110010,
675 0x9214, 0xffffffff, 0x000a0009,
676 0x9218, 0xffffffff, 0x000c000b,
677 0x921c, 0xffffffff, 0x0000000f,
678 0x9220, 0xffffffff, 0x000e000d,
679 0x9224, 0xffffffff, 0x00110010,
680 0x9228, 0xffffffff, 0x000b000a,
681 0x922c, 0xffffffff, 0x000d000c,
682 0x9230, 0xffffffff, 0x00000010,
683 0x9234, 0xffffffff, 0x000f000e,
684 0x9238, 0xffffffff, 0x00120011,
685 0x923c, 0xffffffff, 0x000c000b,
686 0x9240, 0xffffffff, 0x000e000d,
687 0x9244, 0xffffffff, 0x00000011,
688 0x9248, 0xffffffff, 0x0010000f,
689 0x924c, 0xffffffff, 0x00130012,
690 0x9250, 0xffffffff, 0x000d000c,
691 0x9254, 0xffffffff, 0x000f000e,
692 0x9258, 0xffffffff, 0x00100013,
693 0x925c, 0xffffffff, 0x00120011,
694 0x9260, 0xffffffff, 0x00150014,
695 0x9264, 0xffffffff, 0x000e000d,
696 0x9268, 0xffffffff, 0x0010000f,
697 0x926c, 0xffffffff, 0x00000013,
698 0x9270, 0xffffffff, 0x00120011,
699 0x9274, 0xffffffff, 0x00150014,
700 0x9278, 0xffffffff, 0x000f000e,
701 0x927c, 0xffffffff, 0x00110010,
702 0x9280, 0xffffffff, 0x00000014,
703 0x9284, 0xffffffff, 0x00130012,
704 0x9288, 0xffffffff, 0x00160015,
705 0x928c, 0xffffffff, 0x0010000f,
706 0x9290, 0xffffffff, 0x00120011,
707 0x9294, 0xffffffff, 0x00000015,
708 0x9298, 0xffffffff, 0x00140013,
709 0x929c, 0xffffffff, 0x00170016,
710 0x9150, 0xffffffff, 0x96940200,
711 0x8708, 0xffffffff, 0x00900100,
712 0xc478, 0xffffffff, 0x00000080,
713 0xc404, 0xffffffff, 0x0020003f,
714 0x30, 0xffffffff, 0x0000001c,
715 0x34, 0x000f0000, 0x000f0000,
716 0x160c, 0xffffffff, 0x00000100,
717 0x1024, 0xffffffff, 0x00000100,
718 0x102c, 0x00000101, 0x00000000,
719 0x20a8, 0xffffffff, 0x00000104,
720 0x264c, 0x000c0000, 0x000c0000,
721 0x2648, 0x000c0000, 0x000c0000,
722 0x55e4, 0xff000fff, 0x00000100,
723 0x55e8, 0x00000001, 0x00000001,
724 0x2f50, 0x00000001, 0x00000001,
725 0x30cc, 0xc0000fff, 0x00000104,
726 0xc1e4, 0x00000001, 0x00000001,
727 0xd0c0, 0xfffffff0, 0x00000100,
728 0xd8c0, 0xfffffff0, 0x00000100
731 static const u32 pitcairn_mgcg_cgcg_init[] = {
732 0xc400, 0xffffffff, 0xfffffffc,
733 0x802c, 0xffffffff, 0xe0000000,
734 0x9a60, 0xffffffff, 0x00000100,
735 0x92a4, 0xffffffff, 0x00000100,
736 0xc164, 0xffffffff, 0x00000100,
737 0x9774, 0xffffffff, 0x00000100,
738 0x8984, 0xffffffff, 0x06000100,
739 0x8a18, 0xffffffff, 0x00000100,
740 0x92a0, 0xffffffff, 0x00000100,
741 0xc380, 0xffffffff, 0x00000100,
742 0x8b28, 0xffffffff, 0x00000100,
743 0x9144, 0xffffffff, 0x00000100,
744 0x8d88, 0xffffffff, 0x00000100,
745 0x8d8c, 0xffffffff, 0x00000100,
746 0x9030, 0xffffffff, 0x00000100,
747 0x9034, 0xffffffff, 0x00000100,
748 0x9038, 0xffffffff, 0x00000100,
749 0x903c, 0xffffffff, 0x00000100,
750 0xad80, 0xffffffff, 0x00000100,
751 0xac54, 0xffffffff, 0x00000100,
752 0x897c, 0xffffffff, 0x06000100,
753 0x9868, 0xffffffff, 0x00000100,
754 0x9510, 0xffffffff, 0x00000100,
755 0xaf04, 0xffffffff, 0x00000100,
756 0xae04, 0xffffffff, 0x00000100,
757 0x949c, 0xffffffff, 0x00000100,
758 0x802c, 0xffffffff, 0xe0000000,
759 0x9160, 0xffffffff, 0x00010000,
760 0x9164, 0xffffffff, 0x00030002,
761 0x9168, 0xffffffff, 0x00040007,
762 0x916c, 0xffffffff, 0x00060005,
763 0x9170, 0xffffffff, 0x00090008,
764 0x9174, 0xffffffff, 0x00020001,
765 0x9178, 0xffffffff, 0x00040003,
766 0x917c, 0xffffffff, 0x00000007,
767 0x9180, 0xffffffff, 0x00060005,
768 0x9184, 0xffffffff, 0x00090008,
769 0x9188, 0xffffffff, 0x00030002,
770 0x918c, 0xffffffff, 0x00050004,
771 0x9190, 0xffffffff, 0x00000008,
772 0x9194, 0xffffffff, 0x00070006,
773 0x9198, 0xffffffff, 0x000a0009,
774 0x919c, 0xffffffff, 0x00040003,
775 0x91a0, 0xffffffff, 0x00060005,
776 0x91a4, 0xffffffff, 0x00000009,
777 0x91a8, 0xffffffff, 0x00080007,
778 0x91ac, 0xffffffff, 0x000b000a,
779 0x91b0, 0xffffffff, 0x00050004,
780 0x91b4, 0xffffffff, 0x00070006,
781 0x91b8, 0xffffffff, 0x0008000b,
782 0x91bc, 0xffffffff, 0x000a0009,
783 0x91c0, 0xffffffff, 0x000d000c,
784 0x9200, 0xffffffff, 0x00090008,
785 0x9204, 0xffffffff, 0x000b000a,
786 0x9208, 0xffffffff, 0x000c000f,
787 0x920c, 0xffffffff, 0x000e000d,
788 0x9210, 0xffffffff, 0x00110010,
789 0x9214, 0xffffffff, 0x000a0009,
790 0x9218, 0xffffffff, 0x000c000b,
791 0x921c, 0xffffffff, 0x0000000f,
792 0x9220, 0xffffffff, 0x000e000d,
793 0x9224, 0xffffffff, 0x00110010,
794 0x9228, 0xffffffff, 0x000b000a,
795 0x922c, 0xffffffff, 0x000d000c,
796 0x9230, 0xffffffff, 0x00000010,
797 0x9234, 0xffffffff, 0x000f000e,
798 0x9238, 0xffffffff, 0x00120011,
799 0x923c, 0xffffffff, 0x000c000b,
800 0x9240, 0xffffffff, 0x000e000d,
801 0x9244, 0xffffffff, 0x00000011,
802 0x9248, 0xffffffff, 0x0010000f,
803 0x924c, 0xffffffff, 0x00130012,
804 0x9250, 0xffffffff, 0x000d000c,
805 0x9254, 0xffffffff, 0x000f000e,
806 0x9258, 0xffffffff, 0x00100013,
807 0x925c, 0xffffffff, 0x00120011,
808 0x9260, 0xffffffff, 0x00150014,
809 0x9150, 0xffffffff, 0x96940200,
810 0x8708, 0xffffffff, 0x00900100,
811 0xc478, 0xffffffff, 0x00000080,
812 0xc404, 0xffffffff, 0x0020003f,
813 0x30, 0xffffffff, 0x0000001c,
814 0x34, 0x000f0000, 0x000f0000,
815 0x160c, 0xffffffff, 0x00000100,
816 0x1024, 0xffffffff, 0x00000100,
817 0x102c, 0x00000101, 0x00000000,
818 0x20a8, 0xffffffff, 0x00000104,
819 0x55e4, 0xff000fff, 0x00000100,
820 0x55e8, 0x00000001, 0x00000001,
821 0x2f50, 0x00000001, 0x00000001,
822 0x30cc, 0xc0000fff, 0x00000104,
823 0xc1e4, 0x00000001, 0x00000001,
824 0xd0c0, 0xfffffff0, 0x00000100,
825 0xd8c0, 0xfffffff0, 0x00000100
828 static const u32 verde_mgcg_cgcg_init[] = {
829 0xc400, 0xffffffff, 0xfffffffc,
830 0x802c, 0xffffffff, 0xe0000000,
831 0x9a60, 0xffffffff, 0x00000100,
832 0x92a4, 0xffffffff, 0x00000100,
833 0xc164, 0xffffffff, 0x00000100,
834 0x9774, 0xffffffff, 0x00000100,
835 0x8984, 0xffffffff, 0x06000100,
836 0x8a18, 0xffffffff, 0x00000100,
837 0x92a0, 0xffffffff, 0x00000100,
838 0xc380, 0xffffffff, 0x00000100,
839 0x8b28, 0xffffffff, 0x00000100,
840 0x9144, 0xffffffff, 0x00000100,
841 0x8d88, 0xffffffff, 0x00000100,
842 0x8d8c, 0xffffffff, 0x00000100,
843 0x9030, 0xffffffff, 0x00000100,
844 0x9034, 0xffffffff, 0x00000100,
845 0x9038, 0xffffffff, 0x00000100,
846 0x903c, 0xffffffff, 0x00000100,
847 0xad80, 0xffffffff, 0x00000100,
848 0xac54, 0xffffffff, 0x00000100,
849 0x897c, 0xffffffff, 0x06000100,
850 0x9868, 0xffffffff, 0x00000100,
851 0x9510, 0xffffffff, 0x00000100,
852 0xaf04, 0xffffffff, 0x00000100,
853 0xae04, 0xffffffff, 0x00000100,
854 0x949c, 0xffffffff, 0x00000100,
855 0x802c, 0xffffffff, 0xe0000000,
856 0x9160, 0xffffffff, 0x00010000,
857 0x9164, 0xffffffff, 0x00030002,
858 0x9168, 0xffffffff, 0x00040007,
859 0x916c, 0xffffffff, 0x00060005,
860 0x9170, 0xffffffff, 0x00090008,
861 0x9174, 0xffffffff, 0x00020001,
862 0x9178, 0xffffffff, 0x00040003,
863 0x917c, 0xffffffff, 0x00000007,
864 0x9180, 0xffffffff, 0x00060005,
865 0x9184, 0xffffffff, 0x00090008,
866 0x9188, 0xffffffff, 0x00030002,
867 0x918c, 0xffffffff, 0x00050004,
868 0x9190, 0xffffffff, 0x00000008,
869 0x9194, 0xffffffff, 0x00070006,
870 0x9198, 0xffffffff, 0x000a0009,
871 0x919c, 0xffffffff, 0x00040003,
872 0x91a0, 0xffffffff, 0x00060005,
873 0x91a4, 0xffffffff, 0x00000009,
874 0x91a8, 0xffffffff, 0x00080007,
875 0x91ac, 0xffffffff, 0x000b000a,
876 0x91b0, 0xffffffff, 0x00050004,
877 0x91b4, 0xffffffff, 0x00070006,
878 0x91b8, 0xffffffff, 0x0008000b,
879 0x91bc, 0xffffffff, 0x000a0009,
880 0x91c0, 0xffffffff, 0x000d000c,
881 0x9200, 0xffffffff, 0x00090008,
882 0x9204, 0xffffffff, 0x000b000a,
883 0x9208, 0xffffffff, 0x000c000f,
884 0x920c, 0xffffffff, 0x000e000d,
885 0x9210, 0xffffffff, 0x00110010,
886 0x9214, 0xffffffff, 0x000a0009,
887 0x9218, 0xffffffff, 0x000c000b,
888 0x921c, 0xffffffff, 0x0000000f,
889 0x9220, 0xffffffff, 0x000e000d,
890 0x9224, 0xffffffff, 0x00110010,
891 0x9228, 0xffffffff, 0x000b000a,
892 0x922c, 0xffffffff, 0x000d000c,
893 0x9230, 0xffffffff, 0x00000010,
894 0x9234, 0xffffffff, 0x000f000e,
895 0x9238, 0xffffffff, 0x00120011,
896 0x923c, 0xffffffff, 0x000c000b,
897 0x9240, 0xffffffff, 0x000e000d,
898 0x9244, 0xffffffff, 0x00000011,
899 0x9248, 0xffffffff, 0x0010000f,
900 0x924c, 0xffffffff, 0x00130012,
901 0x9250, 0xffffffff, 0x000d000c,
902 0x9254, 0xffffffff, 0x000f000e,
903 0x9258, 0xffffffff, 0x00100013,
904 0x925c, 0xffffffff, 0x00120011,
905 0x9260, 0xffffffff, 0x00150014,
906 0x9150, 0xffffffff, 0x96940200,
907 0x8708, 0xffffffff, 0x00900100,
908 0xc478, 0xffffffff, 0x00000080,
909 0xc404, 0xffffffff, 0x0020003f,
910 0x30, 0xffffffff, 0x0000001c,
911 0x34, 0x000f0000, 0x000f0000,
912 0x160c, 0xffffffff, 0x00000100,
913 0x1024, 0xffffffff, 0x00000100,
914 0x102c, 0x00000101, 0x00000000,
915 0x20a8, 0xffffffff, 0x00000104,
916 0x264c, 0x000c0000, 0x000c0000,
917 0x2648, 0x000c0000, 0x000c0000,
918 0x55e4, 0xff000fff, 0x00000100,
919 0x55e8, 0x00000001, 0x00000001,
920 0x2f50, 0x00000001, 0x00000001,
921 0x30cc, 0xc0000fff, 0x00000104,
922 0xc1e4, 0x00000001, 0x00000001,
923 0xd0c0, 0xfffffff0, 0x00000100,
924 0xd8c0, 0xfffffff0, 0x00000100
927 static const u32 oland_mgcg_cgcg_init[] = {
928 0xc400, 0xffffffff, 0xfffffffc,
929 0x802c, 0xffffffff, 0xe0000000,
930 0x9a60, 0xffffffff, 0x00000100,
931 0x92a4, 0xffffffff, 0x00000100,
932 0xc164, 0xffffffff, 0x00000100,
933 0x9774, 0xffffffff, 0x00000100,
934 0x8984, 0xffffffff, 0x06000100,
935 0x8a18, 0xffffffff, 0x00000100,
936 0x92a0, 0xffffffff, 0x00000100,
937 0xc380, 0xffffffff, 0x00000100,
938 0x8b28, 0xffffffff, 0x00000100,
939 0x9144, 0xffffffff, 0x00000100,
940 0x8d88, 0xffffffff, 0x00000100,
941 0x8d8c, 0xffffffff, 0x00000100,
942 0x9030, 0xffffffff, 0x00000100,
943 0x9034, 0xffffffff, 0x00000100,
944 0x9038, 0xffffffff, 0x00000100,
945 0x903c, 0xffffffff, 0x00000100,
946 0xad80, 0xffffffff, 0x00000100,
947 0xac54, 0xffffffff, 0x00000100,
948 0x897c, 0xffffffff, 0x06000100,
949 0x9868, 0xffffffff, 0x00000100,
950 0x9510, 0xffffffff, 0x00000100,
951 0xaf04, 0xffffffff, 0x00000100,
952 0xae04, 0xffffffff, 0x00000100,
953 0x949c, 0xffffffff, 0x00000100,
954 0x802c, 0xffffffff, 0xe0000000,
955 0x9160, 0xffffffff, 0x00010000,
956 0x9164, 0xffffffff, 0x00030002,
957 0x9168, 0xffffffff, 0x00040007,
958 0x916c, 0xffffffff, 0x00060005,
959 0x9170, 0xffffffff, 0x00090008,
960 0x9174, 0xffffffff, 0x00020001,
961 0x9178, 0xffffffff, 0x00040003,
962 0x917c, 0xffffffff, 0x00000007,
963 0x9180, 0xffffffff, 0x00060005,
964 0x9184, 0xffffffff, 0x00090008,
965 0x9188, 0xffffffff, 0x00030002,
966 0x918c, 0xffffffff, 0x00050004,
967 0x9190, 0xffffffff, 0x00000008,
968 0x9194, 0xffffffff, 0x00070006,
969 0x9198, 0xffffffff, 0x000a0009,
970 0x919c, 0xffffffff, 0x00040003,
971 0x91a0, 0xffffffff, 0x00060005,
972 0x91a4, 0xffffffff, 0x00000009,
973 0x91a8, 0xffffffff, 0x00080007,
974 0x91ac, 0xffffffff, 0x000b000a,
975 0x91b0, 0xffffffff, 0x00050004,
976 0x91b4, 0xffffffff, 0x00070006,
977 0x91b8, 0xffffffff, 0x0008000b,
978 0x91bc, 0xffffffff, 0x000a0009,
979 0x91c0, 0xffffffff, 0x000d000c,
980 0x91c4, 0xffffffff, 0x00060005,
981 0x91c8, 0xffffffff, 0x00080007,
982 0x91cc, 0xffffffff, 0x0000000b,
983 0x91d0, 0xffffffff, 0x000a0009,
984 0x91d4, 0xffffffff, 0x000d000c,
985 0x9150, 0xffffffff, 0x96940200,
986 0x8708, 0xffffffff, 0x00900100,
987 0xc478, 0xffffffff, 0x00000080,
988 0xc404, 0xffffffff, 0x0020003f,
989 0x30, 0xffffffff, 0x0000001c,
990 0x34, 0x000f0000, 0x000f0000,
991 0x160c, 0xffffffff, 0x00000100,
992 0x1024, 0xffffffff, 0x00000100,
993 0x102c, 0x00000101, 0x00000000,
994 0x20a8, 0xffffffff, 0x00000104,
995 0x264c, 0x000c0000, 0x000c0000,
996 0x2648, 0x000c0000, 0x000c0000,
997 0x55e4, 0xff000fff, 0x00000100,
998 0x55e8, 0x00000001, 0x00000001,
999 0x2f50, 0x00000001, 0x00000001,
1000 0x30cc, 0xc0000fff, 0x00000104,
1001 0xc1e4, 0x00000001, 0x00000001,
1002 0xd0c0, 0xfffffff0, 0x00000100,
1003 0xd8c0, 0xfffffff0, 0x00000100
1006 static const u32 hainan_mgcg_cgcg_init[] = {
1007 0xc400, 0xffffffff, 0xfffffffc,
1008 0x802c, 0xffffffff, 0xe0000000,
1009 0x9a60, 0xffffffff, 0x00000100,
1010 0x92a4, 0xffffffff, 0x00000100,
1011 0xc164, 0xffffffff, 0x00000100,
1012 0x9774, 0xffffffff, 0x00000100,
1013 0x8984, 0xffffffff, 0x06000100,
1014 0x8a18, 0xffffffff, 0x00000100,
1015 0x92a0, 0xffffffff, 0x00000100,
1016 0xc380, 0xffffffff, 0x00000100,
1017 0x8b28, 0xffffffff, 0x00000100,
1018 0x9144, 0xffffffff, 0x00000100,
1019 0x8d88, 0xffffffff, 0x00000100,
1020 0x8d8c, 0xffffffff, 0x00000100,
1021 0x9030, 0xffffffff, 0x00000100,
1022 0x9034, 0xffffffff, 0x00000100,
1023 0x9038, 0xffffffff, 0x00000100,
1024 0x903c, 0xffffffff, 0x00000100,
1025 0xad80, 0xffffffff, 0x00000100,
1026 0xac54, 0xffffffff, 0x00000100,
1027 0x897c, 0xffffffff, 0x06000100,
1028 0x9868, 0xffffffff, 0x00000100,
1029 0x9510, 0xffffffff, 0x00000100,
1030 0xaf04, 0xffffffff, 0x00000100,
1031 0xae04, 0xffffffff, 0x00000100,
1032 0x949c, 0xffffffff, 0x00000100,
1033 0x802c, 0xffffffff, 0xe0000000,
1034 0x9160, 0xffffffff, 0x00010000,
1035 0x9164, 0xffffffff, 0x00030002,
1036 0x9168, 0xffffffff, 0x00040007,
1037 0x916c, 0xffffffff, 0x00060005,
1038 0x9170, 0xffffffff, 0x00090008,
1039 0x9174, 0xffffffff, 0x00020001,
1040 0x9178, 0xffffffff, 0x00040003,
1041 0x917c, 0xffffffff, 0x00000007,
1042 0x9180, 0xffffffff, 0x00060005,
1043 0x9184, 0xffffffff, 0x00090008,
1044 0x9188, 0xffffffff, 0x00030002,
1045 0x918c, 0xffffffff, 0x00050004,
1046 0x9190, 0xffffffff, 0x00000008,
1047 0x9194, 0xffffffff, 0x00070006,
1048 0x9198, 0xffffffff, 0x000a0009,
1049 0x919c, 0xffffffff, 0x00040003,
1050 0x91a0, 0xffffffff, 0x00060005,
1051 0x91a4, 0xffffffff, 0x00000009,
1052 0x91a8, 0xffffffff, 0x00080007,
1053 0x91ac, 0xffffffff, 0x000b000a,
1054 0x91b0, 0xffffffff, 0x00050004,
1055 0x91b4, 0xffffffff, 0x00070006,
1056 0x91b8, 0xffffffff, 0x0008000b,
1057 0x91bc, 0xffffffff, 0x000a0009,
1058 0x91c0, 0xffffffff, 0x000d000c,
1059 0x91c4, 0xffffffff, 0x00060005,
1060 0x91c8, 0xffffffff, 0x00080007,
1061 0x91cc, 0xffffffff, 0x0000000b,
1062 0x91d0, 0xffffffff, 0x000a0009,
1063 0x91d4, 0xffffffff, 0x000d000c,
1064 0x9150, 0xffffffff, 0x96940200,
1065 0x8708, 0xffffffff, 0x00900100,
1066 0xc478, 0xffffffff, 0x00000080,
1067 0xc404, 0xffffffff, 0x0020003f,
1068 0x30, 0xffffffff, 0x0000001c,
1069 0x34, 0x000f0000, 0x000f0000,
1070 0x160c, 0xffffffff, 0x00000100,
1071 0x1024, 0xffffffff, 0x00000100,
1072 0x20a8, 0xffffffff, 0x00000104,
1073 0x264c, 0x000c0000, 0x000c0000,
1074 0x2648, 0x000c0000, 0x000c0000,
1075 0x2f50, 0x00000001, 0x00000001,
1076 0x30cc, 0xc0000fff, 0x00000104,
1077 0xc1e4, 0x00000001, 0x00000001,
1078 0xd0c0, 0xfffffff0, 0x00000100,
1079 0xd8c0, 0xfffffff0, 0x00000100
1082 static u32 verde_pg_init[] = {
1083 0x353c, 0xffffffff, 0x40000,
1084 0x3538, 0xffffffff, 0x200010ff,
1085 0x353c, 0xffffffff, 0x0,
1086 0x353c, 0xffffffff, 0x0,
1087 0x353c, 0xffffffff, 0x0,
1088 0x353c, 0xffffffff, 0x0,
1089 0x353c, 0xffffffff, 0x0,
1090 0x353c, 0xffffffff, 0x7007,
1091 0x3538, 0xffffffff, 0x300010ff,
1092 0x353c, 0xffffffff, 0x0,
1093 0x353c, 0xffffffff, 0x0,
1094 0x353c, 0xffffffff, 0x0,
1095 0x353c, 0xffffffff, 0x0,
1096 0x353c, 0xffffffff, 0x0,
1097 0x353c, 0xffffffff, 0x400000,
1098 0x3538, 0xffffffff, 0x100010ff,
1099 0x353c, 0xffffffff, 0x0,
1100 0x353c, 0xffffffff, 0x0,
1101 0x353c, 0xffffffff, 0x0,
1102 0x353c, 0xffffffff, 0x0,
1103 0x353c, 0xffffffff, 0x0,
1104 0x353c, 0xffffffff, 0x120200,
1105 0x3538, 0xffffffff, 0x500010ff,
1106 0x353c, 0xffffffff, 0x0,
1107 0x353c, 0xffffffff, 0x0,
1108 0x353c, 0xffffffff, 0x0,
1109 0x353c, 0xffffffff, 0x0,
1110 0x353c, 0xffffffff, 0x0,
1111 0x353c, 0xffffffff, 0x1e1e16,
1112 0x3538, 0xffffffff, 0x600010ff,
1113 0x353c, 0xffffffff, 0x0,
1114 0x353c, 0xffffffff, 0x0,
1115 0x353c, 0xffffffff, 0x0,
1116 0x353c, 0xffffffff, 0x0,
1117 0x353c, 0xffffffff, 0x0,
1118 0x353c, 0xffffffff, 0x171f1e,
1119 0x3538, 0xffffffff, 0x700010ff,
1120 0x353c, 0xffffffff, 0x0,
1121 0x353c, 0xffffffff, 0x0,
1122 0x353c, 0xffffffff, 0x0,
1123 0x353c, 0xffffffff, 0x0,
1124 0x353c, 0xffffffff, 0x0,
1125 0x353c, 0xffffffff, 0x0,
1126 0x3538, 0xffffffff, 0x9ff,
1127 0x3500, 0xffffffff, 0x0,
1128 0x3504, 0xffffffff, 0x10000800,
1129 0x3504, 0xffffffff, 0xf,
1130 0x3504, 0xffffffff, 0xf,
1131 0x3500, 0xffffffff, 0x4,
1132 0x3504, 0xffffffff, 0x1000051e,
1133 0x3504, 0xffffffff, 0xffff,
1134 0x3504, 0xffffffff, 0xffff,
1135 0x3500, 0xffffffff, 0x8,
1136 0x3504, 0xffffffff, 0x80500,
1137 0x3500, 0xffffffff, 0x12,
1138 0x3504, 0xffffffff, 0x9050c,
1139 0x3500, 0xffffffff, 0x1d,
1140 0x3504, 0xffffffff, 0xb052c,
1141 0x3500, 0xffffffff, 0x2a,
1142 0x3504, 0xffffffff, 0x1053e,
1143 0x3500, 0xffffffff, 0x2d,
1144 0x3504, 0xffffffff, 0x10546,
1145 0x3500, 0xffffffff, 0x30,
1146 0x3504, 0xffffffff, 0xa054e,
1147 0x3500, 0xffffffff, 0x3c,
1148 0x3504, 0xffffffff, 0x1055f,
1149 0x3500, 0xffffffff, 0x3f,
1150 0x3504, 0xffffffff, 0x10567,
1151 0x3500, 0xffffffff, 0x42,
1152 0x3504, 0xffffffff, 0x1056f,
1153 0x3500, 0xffffffff, 0x45,
1154 0x3504, 0xffffffff, 0x10572,
1155 0x3500, 0xffffffff, 0x48,
1156 0x3504, 0xffffffff, 0x20575,
1157 0x3500, 0xffffffff, 0x4c,
1158 0x3504, 0xffffffff, 0x190801,
1159 0x3500, 0xffffffff, 0x67,
1160 0x3504, 0xffffffff, 0x1082a,
1161 0x3500, 0xffffffff, 0x6a,
1162 0x3504, 0xffffffff, 0x1b082d,
1163 0x3500, 0xffffffff, 0x87,
1164 0x3504, 0xffffffff, 0x310851,
1165 0x3500, 0xffffffff, 0xba,
1166 0x3504, 0xffffffff, 0x891,
1167 0x3500, 0xffffffff, 0xbc,
1168 0x3504, 0xffffffff, 0x893,
1169 0x3500, 0xffffffff, 0xbe,
1170 0x3504, 0xffffffff, 0x20895,
1171 0x3500, 0xffffffff, 0xc2,
1172 0x3504, 0xffffffff, 0x20899,
1173 0x3500, 0xffffffff, 0xc6,
1174 0x3504, 0xffffffff, 0x2089d,
1175 0x3500, 0xffffffff, 0xca,
1176 0x3504, 0xffffffff, 0x8a1,
1177 0x3500, 0xffffffff, 0xcc,
1178 0x3504, 0xffffffff, 0x8a3,
1179 0x3500, 0xffffffff, 0xce,
1180 0x3504, 0xffffffff, 0x308a5,
1181 0x3500, 0xffffffff, 0xd3,
1182 0x3504, 0xffffffff, 0x6d08cd,
1183 0x3500, 0xffffffff, 0x142,
1184 0x3504, 0xffffffff, 0x2000095a,
1185 0x3504, 0xffffffff, 0x1,
1186 0x3500, 0xffffffff, 0x144,
1187 0x3504, 0xffffffff, 0x301f095b,
1188 0x3500, 0xffffffff, 0x165,
1189 0x3504, 0xffffffff, 0xc094d,
1190 0x3500, 0xffffffff, 0x173,
1191 0x3504, 0xffffffff, 0xf096d,
1192 0x3500, 0xffffffff, 0x184,
1193 0x3504, 0xffffffff, 0x15097f,
1194 0x3500, 0xffffffff, 0x19b,
1195 0x3504, 0xffffffff, 0xc0998,
1196 0x3500, 0xffffffff, 0x1a9,
1197 0x3504, 0xffffffff, 0x409a7,
1198 0x3500, 0xffffffff, 0x1af,
1199 0x3504, 0xffffffff, 0xcdc,
1200 0x3500, 0xffffffff, 0x1b1,
1201 0x3504, 0xffffffff, 0x800,
1202 0x3508, 0xffffffff, 0x6c9b2000,
1203 0x3510, 0xfc00, 0x2000,
1204 0x3544, 0xffffffff, 0xfc0,
1205 0x28d4, 0x00000100, 0x100
1208 static void si_init_golden_registers(struct radeon_device *rdev)
1210 switch (rdev->family) {
1212 radeon_program_register_sequence(rdev,
1213 tahiti_golden_registers,
1214 (const u32)ARRAY_SIZE(tahiti_golden_registers));
1215 radeon_program_register_sequence(rdev,
1216 tahiti_golden_rlc_registers,
1217 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
1218 radeon_program_register_sequence(rdev,
1219 tahiti_mgcg_cgcg_init,
1220 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1221 radeon_program_register_sequence(rdev,
1222 tahiti_golden_registers2,
1223 (const u32)ARRAY_SIZE(tahiti_golden_registers2));
1226 radeon_program_register_sequence(rdev,
1227 pitcairn_golden_registers,
1228 (const u32)ARRAY_SIZE(pitcairn_golden_registers));
1229 radeon_program_register_sequence(rdev,
1230 pitcairn_golden_rlc_registers,
1231 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
1232 radeon_program_register_sequence(rdev,
1233 pitcairn_mgcg_cgcg_init,
1234 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1237 radeon_program_register_sequence(rdev,
1238 verde_golden_registers,
1239 (const u32)ARRAY_SIZE(verde_golden_registers));
1240 radeon_program_register_sequence(rdev,
1241 verde_golden_rlc_registers,
1242 (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
1243 radeon_program_register_sequence(rdev,
1244 verde_mgcg_cgcg_init,
1245 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
1246 radeon_program_register_sequence(rdev,
1248 (const u32)ARRAY_SIZE(verde_pg_init));
1251 radeon_program_register_sequence(rdev,
1252 oland_golden_registers,
1253 (const u32)ARRAY_SIZE(oland_golden_registers));
1254 radeon_program_register_sequence(rdev,
1255 oland_golden_rlc_registers,
1256 (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
1257 radeon_program_register_sequence(rdev,
1258 oland_mgcg_cgcg_init,
1259 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
1262 radeon_program_register_sequence(rdev,
1263 hainan_golden_registers,
1264 (const u32)ARRAY_SIZE(hainan_golden_registers));
1265 radeon_program_register_sequence(rdev,
1266 hainan_golden_registers2,
1267 (const u32)ARRAY_SIZE(hainan_golden_registers2));
1268 radeon_program_register_sequence(rdev,
1269 hainan_mgcg_cgcg_init,
1270 (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
1278 * si_get_allowed_info_register - fetch the register for the info ioctl
1280 * @rdev: radeon_device pointer
1281 * @reg: register offset in bytes
1282 * @val: register value
1284 * Returns 0 for success or -EINVAL for an invalid register
1287 int si_get_allowed_info_register(struct radeon_device *rdev,
1293 case GRBM_STATUS_SE0:
1294 case GRBM_STATUS_SE1:
1297 case (DMA_STATUS_REG + DMA0_REGISTER_OFFSET):
1298 case (DMA_STATUS_REG + DMA1_REGISTER_OFFSET):
1307 #define PCIE_BUS_CLK 10000
1308 #define TCLK (PCIE_BUS_CLK / 10)
1311 * si_get_xclk - get the xclk
1313 * @rdev: radeon_device pointer
1315 * Returns the reference clock used by the gfx engine
1318 u32 si_get_xclk(struct radeon_device *rdev)
1320 u32 reference_clock = rdev->clock.spll.reference_freq;
1323 tmp = RREG32(CG_CLKPIN_CNTL_2);
1324 if (tmp & MUX_TCLK_TO_XCLK)
1327 tmp = RREG32(CG_CLKPIN_CNTL);
1328 if (tmp & XTALIN_DIVIDE)
1329 return reference_clock / 4;
1331 return reference_clock;
1334 /* get temperature in millidegrees */
1335 int si_get_temp(struct radeon_device *rdev)
1338 int actual_temp = 0;
1340 temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
1346 actual_temp = temp & 0x1ff;
1348 actual_temp = (actual_temp * 1000);
1353 #define TAHITI_IO_MC_REGS_SIZE 36
1355 static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1356 {0x0000006f, 0x03044000},
1357 {0x00000070, 0x0480c018},
1358 {0x00000071, 0x00000040},
1359 {0x00000072, 0x01000000},
1360 {0x00000074, 0x000000ff},
1361 {0x00000075, 0x00143400},
1362 {0x00000076, 0x08ec0800},
1363 {0x00000077, 0x040000cc},
1364 {0x00000079, 0x00000000},
1365 {0x0000007a, 0x21000409},
1366 {0x0000007c, 0x00000000},
1367 {0x0000007d, 0xe8000000},
1368 {0x0000007e, 0x044408a8},
1369 {0x0000007f, 0x00000003},
1370 {0x00000080, 0x00000000},
1371 {0x00000081, 0x01000000},
1372 {0x00000082, 0x02000000},
1373 {0x00000083, 0x00000000},
1374 {0x00000084, 0xe3f3e4f4},
1375 {0x00000085, 0x00052024},
1376 {0x00000087, 0x00000000},
1377 {0x00000088, 0x66036603},
1378 {0x00000089, 0x01000000},
1379 {0x0000008b, 0x1c0a0000},
1380 {0x0000008c, 0xff010000},
1381 {0x0000008e, 0xffffefff},
1382 {0x0000008f, 0xfff3efff},
1383 {0x00000090, 0xfff3efbf},
1384 {0x00000094, 0x00101101},
1385 {0x00000095, 0x00000fff},
1386 {0x00000096, 0x00116fff},
1387 {0x00000097, 0x60010000},
1388 {0x00000098, 0x10010000},
1389 {0x00000099, 0x00006000},
1390 {0x0000009a, 0x00001000},
1391 {0x0000009f, 0x00a77400}
1394 static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1395 {0x0000006f, 0x03044000},
1396 {0x00000070, 0x0480c018},
1397 {0x00000071, 0x00000040},
1398 {0x00000072, 0x01000000},
1399 {0x00000074, 0x000000ff},
1400 {0x00000075, 0x00143400},
1401 {0x00000076, 0x08ec0800},
1402 {0x00000077, 0x040000cc},
1403 {0x00000079, 0x00000000},
1404 {0x0000007a, 0x21000409},
1405 {0x0000007c, 0x00000000},
1406 {0x0000007d, 0xe8000000},
1407 {0x0000007e, 0x044408a8},
1408 {0x0000007f, 0x00000003},
1409 {0x00000080, 0x00000000},
1410 {0x00000081, 0x01000000},
1411 {0x00000082, 0x02000000},
1412 {0x00000083, 0x00000000},
1413 {0x00000084, 0xe3f3e4f4},
1414 {0x00000085, 0x00052024},
1415 {0x00000087, 0x00000000},
1416 {0x00000088, 0x66036603},
1417 {0x00000089, 0x01000000},
1418 {0x0000008b, 0x1c0a0000},
1419 {0x0000008c, 0xff010000},
1420 {0x0000008e, 0xffffefff},
1421 {0x0000008f, 0xfff3efff},
1422 {0x00000090, 0xfff3efbf},
1423 {0x00000094, 0x00101101},
1424 {0x00000095, 0x00000fff},
1425 {0x00000096, 0x00116fff},
1426 {0x00000097, 0x60010000},
1427 {0x00000098, 0x10010000},
1428 {0x00000099, 0x00006000},
1429 {0x0000009a, 0x00001000},
1430 {0x0000009f, 0x00a47400}
1433 static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1434 {0x0000006f, 0x03044000},
1435 {0x00000070, 0x0480c018},
1436 {0x00000071, 0x00000040},
1437 {0x00000072, 0x01000000},
1438 {0x00000074, 0x000000ff},
1439 {0x00000075, 0x00143400},
1440 {0x00000076, 0x08ec0800},
1441 {0x00000077, 0x040000cc},
1442 {0x00000079, 0x00000000},
1443 {0x0000007a, 0x21000409},
1444 {0x0000007c, 0x00000000},
1445 {0x0000007d, 0xe8000000},
1446 {0x0000007e, 0x044408a8},
1447 {0x0000007f, 0x00000003},
1448 {0x00000080, 0x00000000},
1449 {0x00000081, 0x01000000},
1450 {0x00000082, 0x02000000},
1451 {0x00000083, 0x00000000},
1452 {0x00000084, 0xe3f3e4f4},
1453 {0x00000085, 0x00052024},
1454 {0x00000087, 0x00000000},
1455 {0x00000088, 0x66036603},
1456 {0x00000089, 0x01000000},
1457 {0x0000008b, 0x1c0a0000},
1458 {0x0000008c, 0xff010000},
1459 {0x0000008e, 0xffffefff},
1460 {0x0000008f, 0xfff3efff},
1461 {0x00000090, 0xfff3efbf},
1462 {0x00000094, 0x00101101},
1463 {0x00000095, 0x00000fff},
1464 {0x00000096, 0x00116fff},
1465 {0x00000097, 0x60010000},
1466 {0x00000098, 0x10010000},
1467 {0x00000099, 0x00006000},
1468 {0x0000009a, 0x00001000},
1469 {0x0000009f, 0x00a37400}
1472 static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1473 {0x0000006f, 0x03044000},
1474 {0x00000070, 0x0480c018},
1475 {0x00000071, 0x00000040},
1476 {0x00000072, 0x01000000},
1477 {0x00000074, 0x000000ff},
1478 {0x00000075, 0x00143400},
1479 {0x00000076, 0x08ec0800},
1480 {0x00000077, 0x040000cc},
1481 {0x00000079, 0x00000000},
1482 {0x0000007a, 0x21000409},
1483 {0x0000007c, 0x00000000},
1484 {0x0000007d, 0xe8000000},
1485 {0x0000007e, 0x044408a8},
1486 {0x0000007f, 0x00000003},
1487 {0x00000080, 0x00000000},
1488 {0x00000081, 0x01000000},
1489 {0x00000082, 0x02000000},
1490 {0x00000083, 0x00000000},
1491 {0x00000084, 0xe3f3e4f4},
1492 {0x00000085, 0x00052024},
1493 {0x00000087, 0x00000000},
1494 {0x00000088, 0x66036603},
1495 {0x00000089, 0x01000000},
1496 {0x0000008b, 0x1c0a0000},
1497 {0x0000008c, 0xff010000},
1498 {0x0000008e, 0xffffefff},
1499 {0x0000008f, 0xfff3efff},
1500 {0x00000090, 0xfff3efbf},
1501 {0x00000094, 0x00101101},
1502 {0x00000095, 0x00000fff},
1503 {0x00000096, 0x00116fff},
1504 {0x00000097, 0x60010000},
1505 {0x00000098, 0x10010000},
1506 {0x00000099, 0x00006000},
1507 {0x0000009a, 0x00001000},
1508 {0x0000009f, 0x00a17730}
1511 static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1512 {0x0000006f, 0x03044000},
1513 {0x00000070, 0x0480c018},
1514 {0x00000071, 0x00000040},
1515 {0x00000072, 0x01000000},
1516 {0x00000074, 0x000000ff},
1517 {0x00000075, 0x00143400},
1518 {0x00000076, 0x08ec0800},
1519 {0x00000077, 0x040000cc},
1520 {0x00000079, 0x00000000},
1521 {0x0000007a, 0x21000409},
1522 {0x0000007c, 0x00000000},
1523 {0x0000007d, 0xe8000000},
1524 {0x0000007e, 0x044408a8},
1525 {0x0000007f, 0x00000003},
1526 {0x00000080, 0x00000000},
1527 {0x00000081, 0x01000000},
1528 {0x00000082, 0x02000000},
1529 {0x00000083, 0x00000000},
1530 {0x00000084, 0xe3f3e4f4},
1531 {0x00000085, 0x00052024},
1532 {0x00000087, 0x00000000},
1533 {0x00000088, 0x66036603},
1534 {0x00000089, 0x01000000},
1535 {0x0000008b, 0x1c0a0000},
1536 {0x0000008c, 0xff010000},
1537 {0x0000008e, 0xffffefff},
1538 {0x0000008f, 0xfff3efff},
1539 {0x00000090, 0xfff3efbf},
1540 {0x00000094, 0x00101101},
1541 {0x00000095, 0x00000fff},
1542 {0x00000096, 0x00116fff},
1543 {0x00000097, 0x60010000},
1544 {0x00000098, 0x10010000},
1545 {0x00000099, 0x00006000},
1546 {0x0000009a, 0x00001000},
1547 {0x0000009f, 0x00a07730}
1551 int si_mc_load_microcode(struct radeon_device *rdev)
1553 const __be32 *fw_data = NULL;
1554 const __le32 *new_fw_data = NULL;
1556 u32 *io_mc_regs = NULL;
1557 const __le32 *new_io_mc_regs = NULL;
1558 int i, regs_size, ucode_size;
1564 const struct mc_firmware_header_v1_0 *hdr =
1565 (const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
1567 radeon_ucode_print_mc_hdr(&hdr->header);
1568 regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1569 new_io_mc_regs = (const __le32 *)
1570 (rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1571 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1572 new_fw_data = (const __le32 *)
1573 (rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1575 ucode_size = rdev->mc_fw->size / 4;
1577 switch (rdev->family) {
1579 io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1580 regs_size = TAHITI_IO_MC_REGS_SIZE;
1583 io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1584 regs_size = TAHITI_IO_MC_REGS_SIZE;
1588 io_mc_regs = (u32 *)&verde_io_mc_regs;
1589 regs_size = TAHITI_IO_MC_REGS_SIZE;
1592 io_mc_regs = (u32 *)&oland_io_mc_regs;
1593 regs_size = TAHITI_IO_MC_REGS_SIZE;
1596 io_mc_regs = (u32 *)&hainan_io_mc_regs;
1597 regs_size = TAHITI_IO_MC_REGS_SIZE;
1600 fw_data = (const __be32 *)rdev->mc_fw->data;
1603 running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1606 /* reset the engine and set to writable */
1607 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1608 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1610 /* load mc io regs */
1611 for (i = 0; i < regs_size; i++) {
1613 WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1614 WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1616 WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1617 WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1620 /* load the MC ucode */
1621 for (i = 0; i < ucode_size; i++) {
1623 WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1625 WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1628 /* put the engine back into the active state */
1629 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1630 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1631 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1633 /* wait for training to complete */
1634 for (i = 0; i < rdev->usec_timeout; i++) {
1635 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1639 for (i = 0; i < rdev->usec_timeout; i++) {
1640 if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1649 static int si_init_microcode(struct radeon_device *rdev)
1651 const char *chip_name;
1652 const char *new_chip_name;
1653 size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
1654 size_t smc_req_size, mc2_req_size;
1658 bool new_smc = false;
1659 bool si58_fw = false;
1660 bool banks2_fw = false;
1664 switch (rdev->family) {
1666 chip_name = "TAHITI";
1667 new_chip_name = "tahiti";
1668 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1669 me_req_size = SI_PM4_UCODE_SIZE * 4;
1670 ce_req_size = SI_CE_UCODE_SIZE * 4;
1671 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1672 mc_req_size = SI_MC_UCODE_SIZE * 4;
1673 mc2_req_size = TAHITI_MC_UCODE_SIZE * 4;
1674 smc_req_size = ALIGN(TAHITI_SMC_UCODE_SIZE, 4);
1677 chip_name = "PITCAIRN";
1678 if ((rdev->pdev->revision == 0x81) &&
1679 ((rdev->pdev->device == 0x6810) ||
1680 (rdev->pdev->device == 0x6811)))
1682 new_chip_name = "pitcairn";
1683 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1684 me_req_size = SI_PM4_UCODE_SIZE * 4;
1685 ce_req_size = SI_CE_UCODE_SIZE * 4;
1686 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1687 mc_req_size = SI_MC_UCODE_SIZE * 4;
1688 mc2_req_size = PITCAIRN_MC_UCODE_SIZE * 4;
1689 smc_req_size = ALIGN(PITCAIRN_SMC_UCODE_SIZE, 4);
1692 chip_name = "VERDE";
1693 if (((rdev->pdev->device == 0x6820) &&
1694 ((rdev->pdev->revision == 0x81) ||
1695 (rdev->pdev->revision == 0x83))) ||
1696 ((rdev->pdev->device == 0x6821) &&
1697 ((rdev->pdev->revision == 0x83) ||
1698 (rdev->pdev->revision == 0x87))) ||
1699 ((rdev->pdev->revision == 0x87) &&
1700 ((rdev->pdev->device == 0x6823) ||
1701 (rdev->pdev->device == 0x682b))))
1703 new_chip_name = "verde";
1704 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1705 me_req_size = SI_PM4_UCODE_SIZE * 4;
1706 ce_req_size = SI_CE_UCODE_SIZE * 4;
1707 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1708 mc_req_size = SI_MC_UCODE_SIZE * 4;
1709 mc2_req_size = VERDE_MC_UCODE_SIZE * 4;
1710 smc_req_size = ALIGN(VERDE_SMC_UCODE_SIZE, 4);
1713 chip_name = "OLAND";
1714 if (((rdev->pdev->revision == 0x81) &&
1715 ((rdev->pdev->device == 0x6600) ||
1716 (rdev->pdev->device == 0x6604) ||
1717 (rdev->pdev->device == 0x6605) ||
1718 (rdev->pdev->device == 0x6610))) ||
1719 ((rdev->pdev->revision == 0x83) &&
1720 (rdev->pdev->device == 0x6610)))
1722 new_chip_name = "oland";
1723 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1724 me_req_size = SI_PM4_UCODE_SIZE * 4;
1725 ce_req_size = SI_CE_UCODE_SIZE * 4;
1726 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1727 mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
1728 smc_req_size = ALIGN(OLAND_SMC_UCODE_SIZE, 4);
1731 chip_name = "HAINAN";
1732 if (((rdev->pdev->revision == 0x81) &&
1733 (rdev->pdev->device == 0x6660)) ||
1734 ((rdev->pdev->revision == 0x83) &&
1735 ((rdev->pdev->device == 0x6660) ||
1736 (rdev->pdev->device == 0x6663) ||
1737 (rdev->pdev->device == 0x6665) ||
1738 (rdev->pdev->device == 0x6667))))
1740 else if ((rdev->pdev->revision == 0xc3) &&
1741 (rdev->pdev->device == 0x6665))
1743 new_chip_name = "hainan";
1744 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1745 me_req_size = SI_PM4_UCODE_SIZE * 4;
1746 ce_req_size = SI_CE_UCODE_SIZE * 4;
1747 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1748 mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
1749 smc_req_size = ALIGN(HAINAN_SMC_UCODE_SIZE, 4);
1755 /* this memory configuration requires special firmware */
1756 if (((RREG32(MC_SEQ_MISC0) & 0xff000000) >> 24) == 0x58)
1759 DRM_INFO("Loading %s Microcode\n", new_chip_name);
1761 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name);
1762 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1764 snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1765 err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1768 if (rdev->pfp_fw->size != pfp_req_size) {
1769 pr_err("si_cp: Bogus length %zu in firmware \"%s\"\n",
1770 rdev->pfp_fw->size, fw_name);
1775 err = radeon_ucode_validate(rdev->pfp_fw);
1777 pr_err("si_cp: validation failed for firmware \"%s\"\n",
1785 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name);
1786 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1788 snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1789 err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1792 if (rdev->me_fw->size != me_req_size) {
1793 pr_err("si_cp: Bogus length %zu in firmware \"%s\"\n",
1794 rdev->me_fw->size, fw_name);
1798 err = radeon_ucode_validate(rdev->me_fw);
1800 pr_err("si_cp: validation failed for firmware \"%s\"\n",
1808 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name);
1809 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1811 snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1812 err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1815 if (rdev->ce_fw->size != ce_req_size) {
1816 pr_err("si_cp: Bogus length %zu in firmware \"%s\"\n",
1817 rdev->ce_fw->size, fw_name);
1821 err = radeon_ucode_validate(rdev->ce_fw);
1823 pr_err("si_cp: validation failed for firmware \"%s\"\n",
1831 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name);
1832 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1834 snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
1835 err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1838 if (rdev->rlc_fw->size != rlc_req_size) {
1839 pr_err("si_rlc: Bogus length %zu in firmware \"%s\"\n",
1840 rdev->rlc_fw->size, fw_name);
1844 err = radeon_ucode_validate(rdev->rlc_fw);
1846 pr_err("si_cp: validation failed for firmware \"%s\"\n",
1855 snprintf(fw_name, sizeof(fw_name), "radeon/si58_mc.bin");
1857 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name);
1858 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1860 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
1861 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1863 snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1864 err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1868 if ((rdev->mc_fw->size != mc_req_size) &&
1869 (rdev->mc_fw->size != mc2_req_size)) {
1870 pr_err("si_mc: Bogus length %zu in firmware \"%s\"\n",
1871 rdev->mc_fw->size, fw_name);
1874 DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
1876 err = radeon_ucode_validate(rdev->mc_fw);
1878 pr_err("si_cp: validation failed for firmware \"%s\"\n",
1887 snprintf(fw_name, sizeof(fw_name), "radeon/banks_k_2_smc.bin");
1889 snprintf(fw_name, sizeof(fw_name), "radeon/%s_k_smc.bin", new_chip_name);
1891 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
1892 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1894 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1895 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1897 pr_err("smc: error loading firmware \"%s\"\n", fw_name);
1898 release_firmware(rdev->smc_fw);
1899 rdev->smc_fw = NULL;
1901 } else if (rdev->smc_fw->size != smc_req_size) {
1902 pr_err("si_smc: Bogus length %zu in firmware \"%s\"\n",
1903 rdev->smc_fw->size, fw_name);
1907 err = radeon_ucode_validate(rdev->smc_fw);
1909 pr_err("si_cp: validation failed for firmware \"%s\"\n",
1918 rdev->new_fw = false;
1919 } else if (new_fw < 6) {
1920 pr_err("si_fw: mixing new and old firmware!\n");
1923 rdev->new_fw = true;
1928 pr_err("si_cp: Failed to load firmware \"%s\"\n",
1930 release_firmware(rdev->pfp_fw);
1931 rdev->pfp_fw = NULL;
1932 release_firmware(rdev->me_fw);
1934 release_firmware(rdev->ce_fw);
1936 release_firmware(rdev->rlc_fw);
1937 rdev->rlc_fw = NULL;
1938 release_firmware(rdev->mc_fw);
1940 release_firmware(rdev->smc_fw);
1941 rdev->smc_fw = NULL;
1946 /* watermark setup */
1947 static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1948 struct radeon_crtc *radeon_crtc,
1949 struct drm_display_mode *mode,
1950 struct drm_display_mode *other_mode)
1952 u32 tmp, buffer_alloc, i;
1953 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1956 * There are 3 line buffers, each one shared by 2 display controllers.
1957 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1958 * the display controllers. The paritioning is done via one of four
1959 * preset allocations specified in bits 21:20:
1961 * 2 - whole lb, other crtc must be disabled
1963 /* this can get tricky if we have two large displays on a paired group
1964 * of crtcs. Ideally for multiple large displays we'd assign them to
1965 * non-linked crtcs for maximum line buffer allocation.
1967 if (radeon_crtc->base.enabled && mode) {
1972 tmp = 2; /* whole */
1980 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1981 DC_LB_MEMORY_CONFIG(tmp));
1983 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1984 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1985 for (i = 0; i < rdev->usec_timeout; i++) {
1986 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1987 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1992 if (radeon_crtc->base.enabled && mode) {
2002 /* controller not enabled, so no lb used */
2006 static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
2008 u32 tmp = RREG32(MC_SHARED_CHMAP);
2010 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
2033 struct dce6_wm_params {
2034 u32 dram_channels; /* number of dram channels */
2035 u32 yclk; /* bandwidth per dram data pin in kHz */
2036 u32 sclk; /* engine clock in kHz */
2037 u32 disp_clk; /* display clock in kHz */
2038 u32 src_width; /* viewport width */
2039 u32 active_time; /* active display time in ns */
2040 u32 blank_time; /* blank time in ns */
2041 bool interlaced; /* mode is interlaced */
2042 fixed20_12 vsc; /* vertical scale ratio */
2043 u32 num_heads; /* number of active crtcs */
2044 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
2045 u32 lb_size; /* line buffer allocated to pipe */
2046 u32 vtaps; /* vertical scaler taps */
2049 static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
2051 /* Calculate raw DRAM Bandwidth */
2052 fixed20_12 dram_efficiency; /* 0.7 */
2053 fixed20_12 yclk, dram_channels, bandwidth;
2056 a.full = dfixed_const(1000);
2057 yclk.full = dfixed_const(wm->yclk);
2058 yclk.full = dfixed_div(yclk, a);
2059 dram_channels.full = dfixed_const(wm->dram_channels * 4);
2060 a.full = dfixed_const(10);
2061 dram_efficiency.full = dfixed_const(7);
2062 dram_efficiency.full = dfixed_div(dram_efficiency, a);
2063 bandwidth.full = dfixed_mul(dram_channels, yclk);
2064 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
2066 return dfixed_trunc(bandwidth);
2069 static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2071 /* Calculate DRAM Bandwidth and the part allocated to display. */
2072 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
2073 fixed20_12 yclk, dram_channels, bandwidth;
2076 a.full = dfixed_const(1000);
2077 yclk.full = dfixed_const(wm->yclk);
2078 yclk.full = dfixed_div(yclk, a);
2079 dram_channels.full = dfixed_const(wm->dram_channels * 4);
2080 a.full = dfixed_const(10);
2081 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
2082 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
2083 bandwidth.full = dfixed_mul(dram_channels, yclk);
2084 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
2086 return dfixed_trunc(bandwidth);
2089 static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
2091 /* Calculate the display Data return Bandwidth */
2092 fixed20_12 return_efficiency; /* 0.8 */
2093 fixed20_12 sclk, bandwidth;
2096 a.full = dfixed_const(1000);
2097 sclk.full = dfixed_const(wm->sclk);
2098 sclk.full = dfixed_div(sclk, a);
2099 a.full = dfixed_const(10);
2100 return_efficiency.full = dfixed_const(8);
2101 return_efficiency.full = dfixed_div(return_efficiency, a);
2102 a.full = dfixed_const(32);
2103 bandwidth.full = dfixed_mul(a, sclk);
2104 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2106 return dfixed_trunc(bandwidth);
2109 static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
2114 static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
2116 /* Calculate the DMIF Request Bandwidth */
2117 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2118 fixed20_12 disp_clk, sclk, bandwidth;
2119 fixed20_12 a, b1, b2;
2122 a.full = dfixed_const(1000);
2123 disp_clk.full = dfixed_const(wm->disp_clk);
2124 disp_clk.full = dfixed_div(disp_clk, a);
2125 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
2126 b1.full = dfixed_mul(a, disp_clk);
2128 a.full = dfixed_const(1000);
2129 sclk.full = dfixed_const(wm->sclk);
2130 sclk.full = dfixed_div(sclk, a);
2131 a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
2132 b2.full = dfixed_mul(a, sclk);
2134 a.full = dfixed_const(10);
2135 disp_clk_request_efficiency.full = dfixed_const(8);
2136 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2138 min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
2140 a.full = dfixed_const(min_bandwidth);
2141 bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
2143 return dfixed_trunc(bandwidth);
2146 static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
2148 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2149 u32 dram_bandwidth = dce6_dram_bandwidth(wm);
2150 u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
2151 u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
2153 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2156 static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
2158 /* Calculate the display mode Average Bandwidth
2159 * DisplayMode should contain the source and destination dimensions,
2163 fixed20_12 line_time;
2164 fixed20_12 src_width;
2165 fixed20_12 bandwidth;
2168 a.full = dfixed_const(1000);
2169 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2170 line_time.full = dfixed_div(line_time, a);
2171 bpp.full = dfixed_const(wm->bytes_per_pixel);
2172 src_width.full = dfixed_const(wm->src_width);
2173 bandwidth.full = dfixed_mul(src_width, bpp);
2174 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2175 bandwidth.full = dfixed_div(bandwidth, line_time);
2177 return dfixed_trunc(bandwidth);
2180 static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
2182 /* First calcualte the latency in ns */
2183 u32 mc_latency = 2000; /* 2000 ns. */
2184 u32 available_bandwidth = dce6_available_bandwidth(wm);
2185 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2186 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2187 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2188 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2189 (wm->num_heads * cursor_line_pair_return_time);
2190 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2191 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2192 u32 tmp, dmif_size = 12288;
2195 if (wm->num_heads == 0)
2198 a.full = dfixed_const(2);
2199 b.full = dfixed_const(1);
2200 if ((wm->vsc.full > a.full) ||
2201 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2203 ((wm->vsc.full >= a.full) && wm->interlaced))
2204 max_src_lines_per_dst_line = 4;
2206 max_src_lines_per_dst_line = 2;
2208 a.full = dfixed_const(available_bandwidth);
2209 b.full = dfixed_const(wm->num_heads);
2210 a.full = dfixed_div(a, b);
2211 tmp = div_u64((u64) dmif_size * (u64) wm->disp_clk, mc_latency + 512);
2212 tmp = min(dfixed_trunc(a), tmp);
2214 lb_fill_bw = min(tmp, wm->disp_clk * wm->bytes_per_pixel / 1000);
2216 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2217 b.full = dfixed_const(1000);
2218 c.full = dfixed_const(lb_fill_bw);
2219 b.full = dfixed_div(c, b);
2220 a.full = dfixed_div(a, b);
2221 line_fill_time = dfixed_trunc(a);
2223 if (line_fill_time < wm->active_time)
2226 return latency + (line_fill_time - wm->active_time);
2230 static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2232 if (dce6_average_bandwidth(wm) <=
2233 (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
2239 static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
2241 if (dce6_average_bandwidth(wm) <=
2242 (dce6_available_bandwidth(wm) / wm->num_heads))
2248 static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
2250 u32 lb_partitions = wm->lb_size / wm->src_width;
2251 u32 line_time = wm->active_time + wm->blank_time;
2252 u32 latency_tolerant_lines;
2256 a.full = dfixed_const(1);
2257 if (wm->vsc.full > a.full)
2258 latency_tolerant_lines = 1;
2260 if (lb_partitions <= (wm->vtaps + 1))
2261 latency_tolerant_lines = 1;
2263 latency_tolerant_lines = 2;
2266 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2268 if (dce6_latency_watermark(wm) <= latency_hiding)
2274 static void dce6_program_watermarks(struct radeon_device *rdev,
2275 struct radeon_crtc *radeon_crtc,
2276 u32 lb_size, u32 num_heads)
2278 struct drm_display_mode *mode = &radeon_crtc->base.mode;
2279 struct dce6_wm_params wm_low, wm_high;
2283 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2284 u32 priority_a_mark = 0, priority_b_mark = 0;
2285 u32 priority_a_cnt = PRIORITY_OFF;
2286 u32 priority_b_cnt = PRIORITY_OFF;
2287 u32 tmp, arb_control3;
2290 if (radeon_crtc->base.enabled && num_heads && mode) {
2291 active_time = (u32) div_u64((u64)mode->crtc_hdisplay * 1000000,
2293 line_time = (u32) div_u64((u64)mode->crtc_htotal * 1000000,
2295 line_time = min(line_time, (u32)65535);
2299 if (rdev->family == CHIP_ARUBA)
2300 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2302 dram_channels = si_get_number_of_dram_channels(rdev);
2304 /* watermark for high clocks */
2305 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2307 radeon_dpm_get_mclk(rdev, false) * 10;
2309 radeon_dpm_get_sclk(rdev, false) * 10;
2311 wm_high.yclk = rdev->pm.current_mclk * 10;
2312 wm_high.sclk = rdev->pm.current_sclk * 10;
2315 wm_high.disp_clk = mode->clock;
2316 wm_high.src_width = mode->crtc_hdisplay;
2317 wm_high.active_time = active_time;
2318 wm_high.blank_time = line_time - wm_high.active_time;
2319 wm_high.interlaced = false;
2320 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2321 wm_high.interlaced = true;
2322 wm_high.vsc = radeon_crtc->vsc;
2324 if (radeon_crtc->rmx_type != RMX_OFF)
2326 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2327 wm_high.lb_size = lb_size;
2328 wm_high.dram_channels = dram_channels;
2329 wm_high.num_heads = num_heads;
2331 /* watermark for low clocks */
2332 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2334 radeon_dpm_get_mclk(rdev, true) * 10;
2336 radeon_dpm_get_sclk(rdev, true) * 10;
2338 wm_low.yclk = rdev->pm.current_mclk * 10;
2339 wm_low.sclk = rdev->pm.current_sclk * 10;
2342 wm_low.disp_clk = mode->clock;
2343 wm_low.src_width = mode->crtc_hdisplay;
2344 wm_low.active_time = active_time;
2345 wm_low.blank_time = line_time - wm_low.active_time;
2346 wm_low.interlaced = false;
2347 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2348 wm_low.interlaced = true;
2349 wm_low.vsc = radeon_crtc->vsc;
2351 if (radeon_crtc->rmx_type != RMX_OFF)
2353 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2354 wm_low.lb_size = lb_size;
2355 wm_low.dram_channels = dram_channels;
2356 wm_low.num_heads = num_heads;
2358 /* set for high clocks */
2359 latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
2360 /* set for low clocks */
2361 latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
2363 /* possibly force display priority to high */
2364 /* should really do this at mode validation time... */
2365 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2366 !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2367 !dce6_check_latency_hiding(&wm_high) ||
2368 (rdev->disp_priority == 2)) {
2369 DRM_DEBUG_KMS("force priority to high\n");
2370 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2371 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2373 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2374 !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2375 !dce6_check_latency_hiding(&wm_low) ||
2376 (rdev->disp_priority == 2)) {
2377 DRM_DEBUG_KMS("force priority to high\n");
2378 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2379 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2382 a.full = dfixed_const(1000);
2383 b.full = dfixed_const(mode->clock);
2384 b.full = dfixed_div(b, a);
2385 c.full = dfixed_const(latency_watermark_a);
2386 c.full = dfixed_mul(c, b);
2387 c.full = dfixed_mul(c, radeon_crtc->hsc);
2388 c.full = dfixed_div(c, a);
2389 a.full = dfixed_const(16);
2390 c.full = dfixed_div(c, a);
2391 priority_a_mark = dfixed_trunc(c);
2392 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2394 a.full = dfixed_const(1000);
2395 b.full = dfixed_const(mode->clock);
2396 b.full = dfixed_div(b, a);
2397 c.full = dfixed_const(latency_watermark_b);
2398 c.full = dfixed_mul(c, b);
2399 c.full = dfixed_mul(c, radeon_crtc->hsc);
2400 c.full = dfixed_div(c, a);
2401 a.full = dfixed_const(16);
2402 c.full = dfixed_div(c, a);
2403 priority_b_mark = dfixed_trunc(c);
2404 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2406 /* Save number of lines the linebuffer leads before the scanout */
2407 radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
2411 arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2413 tmp &= ~LATENCY_WATERMARK_MASK(3);
2414 tmp |= LATENCY_WATERMARK_MASK(1);
2415 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2416 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2417 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2418 LATENCY_HIGH_WATERMARK(line_time)));
2420 tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2421 tmp &= ~LATENCY_WATERMARK_MASK(3);
2422 tmp |= LATENCY_WATERMARK_MASK(2);
2423 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2424 WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2425 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2426 LATENCY_HIGH_WATERMARK(line_time)));
2427 /* restore original selection */
2428 WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
2430 /* write the priority marks */
2431 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2432 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2434 /* save values for DPM */
2435 radeon_crtc->line_time = line_time;
2436 radeon_crtc->wm_high = latency_watermark_a;
2437 radeon_crtc->wm_low = latency_watermark_b;
2440 void dce6_bandwidth_update(struct radeon_device *rdev)
2442 struct drm_display_mode *mode0 = NULL;
2443 struct drm_display_mode *mode1 = NULL;
2444 u32 num_heads = 0, lb_size;
2447 if (!rdev->mode_info.mode_config_initialized)
2450 radeon_update_display_priority(rdev);
2452 for (i = 0; i < rdev->num_crtc; i++) {
2453 if (rdev->mode_info.crtcs[i]->base.enabled)
2456 for (i = 0; i < rdev->num_crtc; i += 2) {
2457 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2458 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2459 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2460 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2461 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2462 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2469 static void si_tiling_mode_table_init(struct radeon_device *rdev)
2471 u32 *tile = rdev->config.si.tile_mode_array;
2472 const u32 num_tile_mode_states =
2473 ARRAY_SIZE(rdev->config.si.tile_mode_array);
2474 u32 reg_offset, split_equal_to_row_size;
2476 switch (rdev->config.si.mem_row_size_in_kb) {
2478 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2482 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2485 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2489 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2490 tile[reg_offset] = 0;
2492 switch(rdev->family) {
2495 /* non-AA compressed depth or any compressed stencil */
2496 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2497 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2498 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2499 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2500 NUM_BANKS(ADDR_SURF_16_BANK) |
2501 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2502 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2503 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2504 /* 2xAA/4xAA compressed depth only */
2505 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2506 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2507 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2508 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2509 NUM_BANKS(ADDR_SURF_16_BANK) |
2510 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2511 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2512 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2513 /* 8xAA compressed depth only */
2514 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2515 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2516 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2517 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2518 NUM_BANKS(ADDR_SURF_16_BANK) |
2519 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2520 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2521 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2522 /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2523 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2524 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2525 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2526 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2527 NUM_BANKS(ADDR_SURF_16_BANK) |
2528 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2529 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2530 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2531 /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2532 tile[4] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2533 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2534 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2535 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2536 NUM_BANKS(ADDR_SURF_16_BANK) |
2537 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2538 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2539 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2540 /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2541 tile[5] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2542 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2543 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2544 TILE_SPLIT(split_equal_to_row_size) |
2545 NUM_BANKS(ADDR_SURF_16_BANK) |
2546 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2547 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2548 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2549 /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2550 tile[6] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2551 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2552 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2553 TILE_SPLIT(split_equal_to_row_size) |
2554 NUM_BANKS(ADDR_SURF_16_BANK) |
2555 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2556 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2557 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2558 /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2559 tile[7] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2560 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2561 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2562 TILE_SPLIT(split_equal_to_row_size) |
2563 NUM_BANKS(ADDR_SURF_16_BANK) |
2564 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2565 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2566 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2567 /* 1D and 1D Array Surfaces */
2568 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2569 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2570 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2571 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2572 NUM_BANKS(ADDR_SURF_16_BANK) |
2573 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2574 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2575 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2576 /* Displayable maps. */
2577 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2578 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2579 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2580 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2581 NUM_BANKS(ADDR_SURF_16_BANK) |
2582 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2583 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2584 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2586 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2587 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2588 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2589 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2590 NUM_BANKS(ADDR_SURF_16_BANK) |
2591 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2592 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2593 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2594 /* Display 16bpp. */
2595 tile[11] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2596 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2597 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2598 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2599 NUM_BANKS(ADDR_SURF_16_BANK) |
2600 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2601 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2602 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2603 /* Display 32bpp. */
2604 tile[12] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2605 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2606 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2607 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2608 NUM_BANKS(ADDR_SURF_16_BANK) |
2609 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2610 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2611 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2613 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2614 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2615 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2616 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2617 NUM_BANKS(ADDR_SURF_16_BANK) |
2618 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2619 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2620 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2622 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2623 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2624 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2625 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2626 NUM_BANKS(ADDR_SURF_16_BANK) |
2627 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2628 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2629 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2631 tile[15] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2632 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2633 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2634 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2635 NUM_BANKS(ADDR_SURF_16_BANK) |
2636 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2637 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2638 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2640 tile[16] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2641 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2642 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2643 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2644 NUM_BANKS(ADDR_SURF_16_BANK) |
2645 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2646 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2647 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2649 tile[17] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2650 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2651 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2652 TILE_SPLIT(split_equal_to_row_size) |
2653 NUM_BANKS(ADDR_SURF_16_BANK) |
2654 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2655 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2656 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2658 tile[21] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2659 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2660 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2661 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2662 NUM_BANKS(ADDR_SURF_16_BANK) |
2663 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2664 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2665 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2667 tile[22] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2668 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2669 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2670 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2671 NUM_BANKS(ADDR_SURF_16_BANK) |
2672 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2673 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2674 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2676 tile[23] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2677 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2678 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2679 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2680 NUM_BANKS(ADDR_SURF_16_BANK) |
2681 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2682 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2683 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2685 tile[24] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2686 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2687 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2688 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2689 NUM_BANKS(ADDR_SURF_16_BANK) |
2690 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2691 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2692 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2694 tile[25] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2695 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2696 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2697 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2698 NUM_BANKS(ADDR_SURF_8_BANK) |
2699 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2700 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2701 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2703 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2704 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2710 /* non-AA compressed depth or any compressed stencil */
2711 tile[0] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2712 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2713 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2714 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2715 NUM_BANKS(ADDR_SURF_16_BANK) |
2716 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2717 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2718 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2719 /* 2xAA/4xAA compressed depth only */
2720 tile[1] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2721 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2722 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2723 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2724 NUM_BANKS(ADDR_SURF_16_BANK) |
2725 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2726 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2727 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2728 /* 8xAA compressed depth only */
2729 tile[2] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2730 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2731 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2732 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2733 NUM_BANKS(ADDR_SURF_16_BANK) |
2734 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2735 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2736 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2737 /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2738 tile[3] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2739 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2740 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2741 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2742 NUM_BANKS(ADDR_SURF_16_BANK) |
2743 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2744 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2745 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2746 /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2747 tile[4] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2748 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2749 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2750 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2751 NUM_BANKS(ADDR_SURF_16_BANK) |
2752 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2753 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2754 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2755 /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2756 tile[5] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2757 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2758 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2759 TILE_SPLIT(split_equal_to_row_size) |
2760 NUM_BANKS(ADDR_SURF_16_BANK) |
2761 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2762 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2763 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2764 /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2765 tile[6] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2766 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2767 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2768 TILE_SPLIT(split_equal_to_row_size) |
2769 NUM_BANKS(ADDR_SURF_16_BANK) |
2770 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2771 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2772 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2773 /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2774 tile[7] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2775 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2776 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2777 TILE_SPLIT(split_equal_to_row_size) |
2778 NUM_BANKS(ADDR_SURF_16_BANK) |
2779 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2780 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2781 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2782 /* 1D and 1D Array Surfaces */
2783 tile[8] = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2784 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2785 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2786 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2787 NUM_BANKS(ADDR_SURF_16_BANK) |
2788 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2789 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2790 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2791 /* Displayable maps. */
2792 tile[9] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2793 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2794 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2795 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2796 NUM_BANKS(ADDR_SURF_16_BANK) |
2797 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2798 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2799 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2801 tile[10] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2802 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2803 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2804 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2805 NUM_BANKS(ADDR_SURF_16_BANK) |
2806 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2807 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2808 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2809 /* Display 16bpp. */
2810 tile[11] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2811 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2812 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2813 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2814 NUM_BANKS(ADDR_SURF_16_BANK) |
2815 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2816 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2817 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2818 /* Display 32bpp. */
2819 tile[12] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2820 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2821 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2822 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2823 NUM_BANKS(ADDR_SURF_16_BANK) |
2824 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2825 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2826 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2828 tile[13] = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2829 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2830 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2831 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2832 NUM_BANKS(ADDR_SURF_16_BANK) |
2833 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2834 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2835 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2837 tile[14] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2838 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2839 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2840 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2841 NUM_BANKS(ADDR_SURF_16_BANK) |
2842 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2843 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2844 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2846 tile[15] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2847 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2848 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2849 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2850 NUM_BANKS(ADDR_SURF_16_BANK) |
2851 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2852 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2853 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2855 tile[16] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2856 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2857 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2858 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2859 NUM_BANKS(ADDR_SURF_16_BANK) |
2860 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2861 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2862 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2864 tile[17] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2865 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2866 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2867 TILE_SPLIT(split_equal_to_row_size) |
2868 NUM_BANKS(ADDR_SURF_16_BANK) |
2869 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2870 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2871 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2873 tile[21] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2874 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2875 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2876 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2877 NUM_BANKS(ADDR_SURF_16_BANK) |
2878 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2879 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2880 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2882 tile[22] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2883 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2884 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2885 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2886 NUM_BANKS(ADDR_SURF_16_BANK) |
2887 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2888 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2889 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2891 tile[23] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2892 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2893 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2894 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2895 NUM_BANKS(ADDR_SURF_16_BANK) |
2896 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2897 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2898 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2900 tile[24] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2901 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2902 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2903 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2904 NUM_BANKS(ADDR_SURF_16_BANK) |
2905 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2906 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2907 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2909 tile[25] = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2910 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2911 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2912 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2913 NUM_BANKS(ADDR_SURF_8_BANK) |
2914 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2915 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2916 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2918 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++)
2919 WREG32(GB_TILE_MODE0 + (reg_offset * 4), tile[reg_offset]);
2923 DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2927 static void si_select_se_sh(struct radeon_device *rdev,
2928 u32 se_num, u32 sh_num)
2930 u32 data = INSTANCE_BROADCAST_WRITES;
2932 if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
2933 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
2934 else if (se_num == 0xffffffff)
2935 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2936 else if (sh_num == 0xffffffff)
2937 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2939 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2940 WREG32(GRBM_GFX_INDEX, data);
2943 static u32 si_create_bitmask(u32 bit_width)
2947 for (i = 0; i < bit_width; i++) {
2954 static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2958 data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2960 data &= INACTIVE_CUS_MASK;
2963 data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2965 data >>= INACTIVE_CUS_SHIFT;
2967 mask = si_create_bitmask(cu_per_sh);
2969 return ~data & mask;
2972 static void si_setup_spi(struct radeon_device *rdev,
2973 u32 se_num, u32 sh_per_se,
2977 u32 data, mask, active_cu;
2979 for (i = 0; i < se_num; i++) {
2980 for (j = 0; j < sh_per_se; j++) {
2981 si_select_se_sh(rdev, i, j);
2982 data = RREG32(SPI_STATIC_THREAD_MGMT_3);
2983 active_cu = si_get_cu_enabled(rdev, cu_per_sh);
2986 for (k = 0; k < 16; k++) {
2988 if (active_cu & mask) {
2990 WREG32(SPI_STATIC_THREAD_MGMT_3, data);
2996 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2999 static u32 si_get_rb_disabled(struct radeon_device *rdev,
3000 u32 max_rb_num_per_se,
3005 data = RREG32(CC_RB_BACKEND_DISABLE);
3007 data &= BACKEND_DISABLE_MASK;
3010 data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3012 data >>= BACKEND_DISABLE_SHIFT;
3014 mask = si_create_bitmask(max_rb_num_per_se / sh_per_se);
3019 static void si_setup_rb(struct radeon_device *rdev,
3020 u32 se_num, u32 sh_per_se,
3021 u32 max_rb_num_per_se)
3025 u32 disabled_rbs = 0;
3026 u32 enabled_rbs = 0;
3028 for (i = 0; i < se_num; i++) {
3029 for (j = 0; j < sh_per_se; j++) {
3030 si_select_se_sh(rdev, i, j);
3031 data = si_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
3032 disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
3035 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3038 for (i = 0; i < max_rb_num_per_se * se_num; i++) {
3039 if (!(disabled_rbs & mask))
3040 enabled_rbs |= mask;
3044 rdev->config.si.backend_enable_mask = enabled_rbs;
3046 for (i = 0; i < se_num; i++) {
3047 si_select_se_sh(rdev, i, 0xffffffff);
3049 for (j = 0; j < sh_per_se; j++) {
3050 switch (enabled_rbs & 3) {
3052 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3055 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3059 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3064 WREG32(PA_SC_RASTER_CONFIG, data);
3066 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3069 static void si_gpu_init(struct radeon_device *rdev)
3071 u32 gb_addr_config = 0;
3074 u32 hdp_host_path_cntl;
3078 switch (rdev->family) {
3080 rdev->config.si.max_shader_engines = 2;
3081 rdev->config.si.max_tile_pipes = 12;
3082 rdev->config.si.max_cu_per_sh = 8;
3083 rdev->config.si.max_sh_per_se = 2;
3084 rdev->config.si.max_backends_per_se = 4;
3085 rdev->config.si.max_texture_channel_caches = 12;
3086 rdev->config.si.max_gprs = 256;
3087 rdev->config.si.max_gs_threads = 32;
3088 rdev->config.si.max_hw_contexts = 8;
3090 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3091 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
3092 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3093 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3094 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
3097 rdev->config.si.max_shader_engines = 2;
3098 rdev->config.si.max_tile_pipes = 8;
3099 rdev->config.si.max_cu_per_sh = 5;
3100 rdev->config.si.max_sh_per_se = 2;
3101 rdev->config.si.max_backends_per_se = 4;
3102 rdev->config.si.max_texture_channel_caches = 8;
3103 rdev->config.si.max_gprs = 256;
3104 rdev->config.si.max_gs_threads = 32;
3105 rdev->config.si.max_hw_contexts = 8;
3107 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3108 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
3109 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3110 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3111 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
3115 rdev->config.si.max_shader_engines = 1;
3116 rdev->config.si.max_tile_pipes = 4;
3117 rdev->config.si.max_cu_per_sh = 5;
3118 rdev->config.si.max_sh_per_se = 2;
3119 rdev->config.si.max_backends_per_se = 4;
3120 rdev->config.si.max_texture_channel_caches = 4;
3121 rdev->config.si.max_gprs = 256;
3122 rdev->config.si.max_gs_threads = 32;
3123 rdev->config.si.max_hw_contexts = 8;
3125 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3126 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3127 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3128 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3129 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
3132 rdev->config.si.max_shader_engines = 1;
3133 rdev->config.si.max_tile_pipes = 4;
3134 rdev->config.si.max_cu_per_sh = 6;
3135 rdev->config.si.max_sh_per_se = 1;
3136 rdev->config.si.max_backends_per_se = 2;
3137 rdev->config.si.max_texture_channel_caches = 4;
3138 rdev->config.si.max_gprs = 256;
3139 rdev->config.si.max_gs_threads = 16;
3140 rdev->config.si.max_hw_contexts = 8;
3142 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3143 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3144 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3145 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3146 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
3149 rdev->config.si.max_shader_engines = 1;
3150 rdev->config.si.max_tile_pipes = 4;
3151 rdev->config.si.max_cu_per_sh = 5;
3152 rdev->config.si.max_sh_per_se = 1;
3153 rdev->config.si.max_backends_per_se = 1;
3154 rdev->config.si.max_texture_channel_caches = 2;
3155 rdev->config.si.max_gprs = 256;
3156 rdev->config.si.max_gs_threads = 16;
3157 rdev->config.si.max_hw_contexts = 8;
3159 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3160 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3161 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3162 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3163 gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
3167 /* Initialize HDP */
3168 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3169 WREG32((0x2c14 + j), 0x00000000);
3170 WREG32((0x2c18 + j), 0x00000000);
3171 WREG32((0x2c1c + j), 0x00000000);
3172 WREG32((0x2c20 + j), 0x00000000);
3173 WREG32((0x2c24 + j), 0x00000000);
3176 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3177 WREG32(SRBM_INT_CNTL, 1);
3178 WREG32(SRBM_INT_ACK, 1);
3180 evergreen_fix_pci_max_read_req_size(rdev);
3182 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3184 RREG32(MC_SHARED_CHMAP);
3185 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3187 rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
3188 rdev->config.si.mem_max_burst_length_bytes = 256;
3189 tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3190 rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3191 if (rdev->config.si.mem_row_size_in_kb > 4)
3192 rdev->config.si.mem_row_size_in_kb = 4;
3193 /* XXX use MC settings? */
3194 rdev->config.si.shader_engine_tile_size = 32;
3195 rdev->config.si.num_gpus = 1;
3196 rdev->config.si.multi_gpu_tile_size = 64;
3198 /* fix up row size */
3199 gb_addr_config &= ~ROW_SIZE_MASK;
3200 switch (rdev->config.si.mem_row_size_in_kb) {
3203 gb_addr_config |= ROW_SIZE(0);
3206 gb_addr_config |= ROW_SIZE(1);
3209 gb_addr_config |= ROW_SIZE(2);
3213 /* setup tiling info dword. gb_addr_config is not adequate since it does
3214 * not have bank info, so create a custom tiling dword.
3215 * bits 3:0 num_pipes
3216 * bits 7:4 num_banks
3217 * bits 11:8 group_size
3218 * bits 15:12 row_size
3220 rdev->config.si.tile_config = 0;
3221 switch (rdev->config.si.num_tile_pipes) {
3223 rdev->config.si.tile_config |= (0 << 0);
3226 rdev->config.si.tile_config |= (1 << 0);
3229 rdev->config.si.tile_config |= (2 << 0);
3233 /* XXX what about 12? */
3234 rdev->config.si.tile_config |= (3 << 0);
3237 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3238 case 0: /* four banks */
3239 rdev->config.si.tile_config |= 0 << 4;
3241 case 1: /* eight banks */
3242 rdev->config.si.tile_config |= 1 << 4;
3244 case 2: /* sixteen banks */
3246 rdev->config.si.tile_config |= 2 << 4;
3249 rdev->config.si.tile_config |=
3250 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3251 rdev->config.si.tile_config |=
3252 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3254 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3255 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3256 WREG32(DMIF_ADDR_CALC, gb_addr_config);
3257 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3258 WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
3259 WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
3260 if (rdev->has_uvd) {
3261 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3262 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3263 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3266 si_tiling_mode_table_init(rdev);
3268 si_setup_rb(rdev, rdev->config.si.max_shader_engines,
3269 rdev->config.si.max_sh_per_se,
3270 rdev->config.si.max_backends_per_se);
3272 si_setup_spi(rdev, rdev->config.si.max_shader_engines,
3273 rdev->config.si.max_sh_per_se,
3274 rdev->config.si.max_cu_per_sh);
3276 rdev->config.si.active_cus = 0;
3277 for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
3278 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
3279 rdev->config.si.active_cus +=
3280 hweight32(si_get_cu_active_bitmap(rdev, i, j));
3284 /* set HW defaults for 3D engine */
3285 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3286 ROQ_IB2_START(0x2b)));
3287 WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3289 sx_debug_1 = RREG32(SX_DEBUG_1);
3290 WREG32(SX_DEBUG_1, sx_debug_1);
3292 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3294 WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
3295 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
3296 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
3297 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
3299 WREG32(VGT_NUM_INSTANCES, 1);
3301 WREG32(CP_PERFMON_CNTL, 0);
3303 WREG32(SQ_CONFIG, 0);
3305 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3306 FORCE_EOV_MAX_REZ_CNT(255)));
3308 WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3309 AUTO_INVLD_EN(ES_AND_GS_AUTO));
3311 WREG32(VGT_GS_VERTEX_REUSE, 16);
3312 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3314 WREG32(CB_PERFCOUNTER0_SELECT0, 0);
3315 WREG32(CB_PERFCOUNTER0_SELECT1, 0);
3316 WREG32(CB_PERFCOUNTER1_SELECT0, 0);
3317 WREG32(CB_PERFCOUNTER1_SELECT1, 0);
3318 WREG32(CB_PERFCOUNTER2_SELECT0, 0);
3319 WREG32(CB_PERFCOUNTER2_SELECT1, 0);
3320 WREG32(CB_PERFCOUNTER3_SELECT0, 0);
3321 WREG32(CB_PERFCOUNTER3_SELECT1, 0);
3323 tmp = RREG32(HDP_MISC_CNTL);
3324 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3325 WREG32(HDP_MISC_CNTL, tmp);
3327 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3328 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3330 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3336 * GPU scratch registers helpers function.
3338 static void si_scratch_init(struct radeon_device *rdev)
3342 rdev->scratch.num_reg = 7;
3343 rdev->scratch.reg_base = SCRATCH_REG0;
3344 for (i = 0; i < rdev->scratch.num_reg; i++) {
3345 rdev->scratch.free[i] = true;
3346 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3350 void si_fence_ring_emit(struct radeon_device *rdev,
3351 struct radeon_fence *fence)
3353 struct radeon_ring *ring = &rdev->ring[fence->ring];
3354 u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3356 /* flush read cache over gart */
3357 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3358 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3359 radeon_ring_write(ring, 0);
3360 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3361 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3362 PACKET3_TC_ACTION_ENA |
3363 PACKET3_SH_KCACHE_ACTION_ENA |
3364 PACKET3_SH_ICACHE_ACTION_ENA);
3365 radeon_ring_write(ring, 0xFFFFFFFF);
3366 radeon_ring_write(ring, 0);
3367 radeon_ring_write(ring, 10); /* poll interval */
3368 /* EVENT_WRITE_EOP - flush caches, send int */
3369 radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3370 radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
3371 radeon_ring_write(ring, lower_32_bits(addr));
3372 radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
3373 radeon_ring_write(ring, fence->seq);
3374 radeon_ring_write(ring, 0);
3380 void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3382 struct radeon_ring *ring = &rdev->ring[ib->ring];
3383 unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
3386 if (ib->is_const_ib) {
3387 /* set switch buffer packet before const IB */
3388 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3389 radeon_ring_write(ring, 0);
3391 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3394 if (ring->rptr_save_reg) {
3395 next_rptr = ring->wptr + 3 + 4 + 8;
3396 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3397 radeon_ring_write(ring, ((ring->rptr_save_reg -
3398 PACKET3_SET_CONFIG_REG_START) >> 2));
3399 radeon_ring_write(ring, next_rptr);
3400 } else if (rdev->wb.enabled) {
3401 next_rptr = ring->wptr + 5 + 4 + 8;
3402 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3403 radeon_ring_write(ring, (1 << 8));
3404 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3405 radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
3406 radeon_ring_write(ring, next_rptr);
3409 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3412 radeon_ring_write(ring, header);
3413 radeon_ring_write(ring,
3417 (ib->gpu_addr & 0xFFFFFFFC));
3418 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3419 radeon_ring_write(ring, ib->length_dw | (vm_id << 24));
3421 if (!ib->is_const_ib) {
3422 /* flush read cache over gart for this vmid */
3423 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3424 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3425 radeon_ring_write(ring, vm_id);
3426 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3427 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3428 PACKET3_TC_ACTION_ENA |
3429 PACKET3_SH_KCACHE_ACTION_ENA |
3430 PACKET3_SH_ICACHE_ACTION_ENA);
3431 radeon_ring_write(ring, 0xFFFFFFFF);
3432 radeon_ring_write(ring, 0);
3433 radeon_ring_write(ring, 10); /* poll interval */
3440 static void si_cp_enable(struct radeon_device *rdev, bool enable)
3443 WREG32(CP_ME_CNTL, 0);
3445 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3446 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3447 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3448 WREG32(SCRATCH_UMSK, 0);
3449 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3450 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3451 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3456 static int si_cp_load_microcode(struct radeon_device *rdev)
3460 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3463 si_cp_enable(rdev, false);
3466 const struct gfx_firmware_header_v1_0 *pfp_hdr =
3467 (const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
3468 const struct gfx_firmware_header_v1_0 *ce_hdr =
3469 (const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
3470 const struct gfx_firmware_header_v1_0 *me_hdr =
3471 (const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
3472 const __le32 *fw_data;
3475 radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
3476 radeon_ucode_print_gfx_hdr(&ce_hdr->header);
3477 radeon_ucode_print_gfx_hdr(&me_hdr->header);
3480 fw_data = (const __le32 *)
3481 (rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3482 fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3483 WREG32(CP_PFP_UCODE_ADDR, 0);
3484 for (i = 0; i < fw_size; i++)
3485 WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
3486 WREG32(CP_PFP_UCODE_ADDR, 0);
3489 fw_data = (const __le32 *)
3490 (rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3491 fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3492 WREG32(CP_CE_UCODE_ADDR, 0);
3493 for (i = 0; i < fw_size; i++)
3494 WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
3495 WREG32(CP_CE_UCODE_ADDR, 0);
3498 fw_data = (const __be32 *)
3499 (rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3500 fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3501 WREG32(CP_ME_RAM_WADDR, 0);
3502 for (i = 0; i < fw_size; i++)
3503 WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
3504 WREG32(CP_ME_RAM_WADDR, 0);
3506 const __be32 *fw_data;
3509 fw_data = (const __be32 *)rdev->pfp_fw->data;
3510 WREG32(CP_PFP_UCODE_ADDR, 0);
3511 for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
3512 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3513 WREG32(CP_PFP_UCODE_ADDR, 0);
3516 fw_data = (const __be32 *)rdev->ce_fw->data;
3517 WREG32(CP_CE_UCODE_ADDR, 0);
3518 for (i = 0; i < SI_CE_UCODE_SIZE; i++)
3519 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3520 WREG32(CP_CE_UCODE_ADDR, 0);
3523 fw_data = (const __be32 *)rdev->me_fw->data;
3524 WREG32(CP_ME_RAM_WADDR, 0);
3525 for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
3526 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3527 WREG32(CP_ME_RAM_WADDR, 0);
3530 WREG32(CP_PFP_UCODE_ADDR, 0);
3531 WREG32(CP_CE_UCODE_ADDR, 0);
3532 WREG32(CP_ME_RAM_WADDR, 0);
3533 WREG32(CP_ME_RAM_RADDR, 0);
3537 static int si_cp_start(struct radeon_device *rdev)
3539 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3542 r = radeon_ring_lock(rdev, ring, 7 + 4);
3544 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3548 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3549 radeon_ring_write(ring, 0x1);
3550 radeon_ring_write(ring, 0x0);
3551 radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
3552 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3553 radeon_ring_write(ring, 0);
3554 radeon_ring_write(ring, 0);
3556 /* init the CE partitions */
3557 radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3558 radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3559 radeon_ring_write(ring, 0xc000);
3560 radeon_ring_write(ring, 0xe000);
3561 radeon_ring_unlock_commit(rdev, ring, false);
3563 si_cp_enable(rdev, true);
3565 r = radeon_ring_lock(rdev, ring, si_default_size + 10);
3567 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3571 /* setup clear context state */
3572 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3573 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3575 for (i = 0; i < si_default_size; i++)
3576 radeon_ring_write(ring, si_default_state[i]);
3578 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3579 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3581 /* set clear context state */
3582 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3583 radeon_ring_write(ring, 0);
3585 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3586 radeon_ring_write(ring, 0x00000316);
3587 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3588 radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3590 radeon_ring_unlock_commit(rdev, ring, false);
3592 for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
3593 ring = &rdev->ring[i];
3594 r = radeon_ring_lock(rdev, ring, 2);
3596 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3600 /* clear the compute context state */
3601 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
3602 radeon_ring_write(ring, 0);
3604 radeon_ring_unlock_commit(rdev, ring, false);
3610 static void si_cp_fini(struct radeon_device *rdev)
3612 struct radeon_ring *ring;
3613 si_cp_enable(rdev, false);
3615 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3616 radeon_ring_fini(rdev, ring);
3617 radeon_scratch_free(rdev, ring->rptr_save_reg);
3619 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3620 radeon_ring_fini(rdev, ring);
3621 radeon_scratch_free(rdev, ring->rptr_save_reg);
3623 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3624 radeon_ring_fini(rdev, ring);
3625 radeon_scratch_free(rdev, ring->rptr_save_reg);
3628 static int si_cp_resume(struct radeon_device *rdev)
3630 struct radeon_ring *ring;
3635 si_enable_gui_idle_interrupt(rdev, false);
3637 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3638 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3640 /* Set the write pointer delay */
3641 WREG32(CP_RB_WPTR_DELAY, 0);
3643 WREG32(CP_DEBUG, 0);
3644 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3646 /* ring 0 - compute and gfx */
3647 /* Set ring buffer size */
3648 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3649 rb_bufsz = order_base_2(ring->ring_size / 8);
3650 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3652 tmp |= BUF_SWAP_32BIT;
3654 WREG32(CP_RB0_CNTL, tmp);
3656 /* Initialize the ring buffer's read and write pointers */
3657 WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3659 WREG32(CP_RB0_WPTR, ring->wptr);
3661 /* set the wb address whether it's enabled or not */
3662 WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3663 WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3665 if (rdev->wb.enabled)
3666 WREG32(SCRATCH_UMSK, 0xff);
3668 tmp |= RB_NO_UPDATE;
3669 WREG32(SCRATCH_UMSK, 0);
3673 WREG32(CP_RB0_CNTL, tmp);
3675 WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
3677 /* ring1 - compute only */
3678 /* Set ring buffer size */
3679 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3680 rb_bufsz = order_base_2(ring->ring_size / 8);
3681 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3683 tmp |= BUF_SWAP_32BIT;
3685 WREG32(CP_RB1_CNTL, tmp);
3687 /* Initialize the ring buffer's read and write pointers */
3688 WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
3690 WREG32(CP_RB1_WPTR, ring->wptr);
3692 /* set the wb address whether it's enabled or not */
3693 WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
3694 WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
3697 WREG32(CP_RB1_CNTL, tmp);
3699 WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
3701 /* ring2 - compute only */
3702 /* Set ring buffer size */
3703 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3704 rb_bufsz = order_base_2(ring->ring_size / 8);
3705 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3707 tmp |= BUF_SWAP_32BIT;
3709 WREG32(CP_RB2_CNTL, tmp);
3711 /* Initialize the ring buffer's read and write pointers */
3712 WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
3714 WREG32(CP_RB2_WPTR, ring->wptr);
3716 /* set the wb address whether it's enabled or not */
3717 WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
3718 WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
3721 WREG32(CP_RB2_CNTL, tmp);
3723 WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
3725 /* start the rings */
3727 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3728 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3729 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3730 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3732 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3733 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3734 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3737 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3739 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3741 r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3743 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3746 si_enable_gui_idle_interrupt(rdev, true);
3748 if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3749 radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
3754 u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
3760 tmp = RREG32(GRBM_STATUS);
3761 if (tmp & (PA_BUSY | SC_BUSY |
3762 BCI_BUSY | SX_BUSY |
3763 TA_BUSY | VGT_BUSY |
3765 GDS_BUSY | SPI_BUSY |
3766 IA_BUSY | IA_BUSY_NO_DMA))
3767 reset_mask |= RADEON_RESET_GFX;
3769 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3770 CP_BUSY | CP_COHERENCY_BUSY))
3771 reset_mask |= RADEON_RESET_CP;
3773 if (tmp & GRBM_EE_BUSY)
3774 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3777 tmp = RREG32(GRBM_STATUS2);
3778 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3779 reset_mask |= RADEON_RESET_RLC;
3781 /* DMA_STATUS_REG 0 */
3782 tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3783 if (!(tmp & DMA_IDLE))
3784 reset_mask |= RADEON_RESET_DMA;
3786 /* DMA_STATUS_REG 1 */
3787 tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3788 if (!(tmp & DMA_IDLE))
3789 reset_mask |= RADEON_RESET_DMA1;
3792 tmp = RREG32(SRBM_STATUS2);
3794 reset_mask |= RADEON_RESET_DMA;
3796 if (tmp & DMA1_BUSY)
3797 reset_mask |= RADEON_RESET_DMA1;
3800 tmp = RREG32(SRBM_STATUS);
3803 reset_mask |= RADEON_RESET_IH;
3806 reset_mask |= RADEON_RESET_SEM;
3808 if (tmp & GRBM_RQ_PENDING)
3809 reset_mask |= RADEON_RESET_GRBM;
3812 reset_mask |= RADEON_RESET_VMC;
3814 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3815 MCC_BUSY | MCD_BUSY))
3816 reset_mask |= RADEON_RESET_MC;
3818 if (evergreen_is_display_hung(rdev))
3819 reset_mask |= RADEON_RESET_DISPLAY;
3822 tmp = RREG32(VM_L2_STATUS);
3824 reset_mask |= RADEON_RESET_VMC;
3826 /* Skip MC reset as it's mostly likely not hung, just busy */
3827 if (reset_mask & RADEON_RESET_MC) {
3828 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3829 reset_mask &= ~RADEON_RESET_MC;
3835 static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3837 struct evergreen_mc_save save;
3838 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3841 if (reset_mask == 0)
3844 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3846 evergreen_print_gpu_status_regs(rdev);
3847 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
3848 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3849 dev_info(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3850 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3859 /* Disable CP parsing/prefetching */
3860 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3862 if (reset_mask & RADEON_RESET_DMA) {
3864 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3865 tmp &= ~DMA_RB_ENABLE;
3866 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
3868 if (reset_mask & RADEON_RESET_DMA1) {
3870 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3871 tmp &= ~DMA_RB_ENABLE;
3872 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3877 evergreen_mc_stop(rdev, &save);
3878 if (evergreen_mc_wait_for_idle(rdev)) {
3879 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3882 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3883 grbm_soft_reset = SOFT_RESET_CB |
3897 if (reset_mask & RADEON_RESET_CP) {
3898 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3900 srbm_soft_reset |= SOFT_RESET_GRBM;
3903 if (reset_mask & RADEON_RESET_DMA)
3904 srbm_soft_reset |= SOFT_RESET_DMA;
3906 if (reset_mask & RADEON_RESET_DMA1)
3907 srbm_soft_reset |= SOFT_RESET_DMA1;
3909 if (reset_mask & RADEON_RESET_DISPLAY)
3910 srbm_soft_reset |= SOFT_RESET_DC;
3912 if (reset_mask & RADEON_RESET_RLC)
3913 grbm_soft_reset |= SOFT_RESET_RLC;
3915 if (reset_mask & RADEON_RESET_SEM)
3916 srbm_soft_reset |= SOFT_RESET_SEM;
3918 if (reset_mask & RADEON_RESET_IH)
3919 srbm_soft_reset |= SOFT_RESET_IH;
3921 if (reset_mask & RADEON_RESET_GRBM)
3922 srbm_soft_reset |= SOFT_RESET_GRBM;
3924 if (reset_mask & RADEON_RESET_VMC)
3925 srbm_soft_reset |= SOFT_RESET_VMC;
3927 if (reset_mask & RADEON_RESET_MC)
3928 srbm_soft_reset |= SOFT_RESET_MC;
3930 if (grbm_soft_reset) {
3931 tmp = RREG32(GRBM_SOFT_RESET);
3932 tmp |= grbm_soft_reset;
3933 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3934 WREG32(GRBM_SOFT_RESET, tmp);
3935 tmp = RREG32(GRBM_SOFT_RESET);
3939 tmp &= ~grbm_soft_reset;
3940 WREG32(GRBM_SOFT_RESET, tmp);
3941 tmp = RREG32(GRBM_SOFT_RESET);
3944 if (srbm_soft_reset) {
3945 tmp = RREG32(SRBM_SOFT_RESET);
3946 tmp |= srbm_soft_reset;
3947 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3948 WREG32(SRBM_SOFT_RESET, tmp);
3949 tmp = RREG32(SRBM_SOFT_RESET);
3953 tmp &= ~srbm_soft_reset;
3954 WREG32(SRBM_SOFT_RESET, tmp);
3955 tmp = RREG32(SRBM_SOFT_RESET);
3958 /* Wait a little for things to settle down */
3961 evergreen_mc_resume(rdev, &save);
3964 evergreen_print_gpu_status_regs(rdev);
3967 static void si_set_clk_bypass_mode(struct radeon_device *rdev)
3971 tmp = RREG32(CG_SPLL_FUNC_CNTL);
3972 tmp |= SPLL_BYPASS_EN;
3973 WREG32(CG_SPLL_FUNC_CNTL, tmp);
3975 tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
3976 tmp |= SPLL_CTLREQ_CHG;
3977 WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
3979 for (i = 0; i < rdev->usec_timeout; i++) {
3980 if (RREG32(SPLL_STATUS) & SPLL_CHG_STATUS)
3985 tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
3986 tmp &= ~(SPLL_CTLREQ_CHG | SCLK_MUX_UPDATE);
3987 WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
3989 tmp = RREG32(MPLL_CNTL_MODE);
3990 tmp &= ~MPLL_MCLK_SEL;
3991 WREG32(MPLL_CNTL_MODE, tmp);
3994 static void si_spll_powerdown(struct radeon_device *rdev)
3998 tmp = RREG32(SPLL_CNTL_MODE);
3999 tmp |= SPLL_SW_DIR_CONTROL;
4000 WREG32(SPLL_CNTL_MODE, tmp);
4002 tmp = RREG32(CG_SPLL_FUNC_CNTL);
4004 WREG32(CG_SPLL_FUNC_CNTL, tmp);
4006 tmp = RREG32(CG_SPLL_FUNC_CNTL);
4008 WREG32(CG_SPLL_FUNC_CNTL, tmp);
4010 tmp = RREG32(SPLL_CNTL_MODE);
4011 tmp &= ~SPLL_SW_DIR_CONTROL;
4012 WREG32(SPLL_CNTL_MODE, tmp);
4015 static void si_gpu_pci_config_reset(struct radeon_device *rdev)
4017 struct evergreen_mc_save save;
4020 dev_info(rdev->dev, "GPU pci config reset\n");
4028 /* Disable CP parsing/prefetching */
4029 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4031 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
4032 tmp &= ~DMA_RB_ENABLE;
4033 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
4035 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
4036 tmp &= ~DMA_RB_ENABLE;
4037 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
4038 /* XXX other engines? */
4040 /* halt the rlc, disable cp internal ints */
4045 /* disable mem access */
4046 evergreen_mc_stop(rdev, &save);
4047 if (evergreen_mc_wait_for_idle(rdev)) {
4048 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
4051 /* set mclk/sclk to bypass */
4052 si_set_clk_bypass_mode(rdev);
4053 /* powerdown spll */
4054 si_spll_powerdown(rdev);
4056 pci_clear_master(rdev->pdev);
4058 radeon_pci_config_reset(rdev);
4059 /* wait for asic to come out of reset */
4060 for (i = 0; i < rdev->usec_timeout; i++) {
4061 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
4067 int si_asic_reset(struct radeon_device *rdev, bool hard)
4072 si_gpu_pci_config_reset(rdev);
4076 reset_mask = si_gpu_check_soft_reset(rdev);
4079 r600_set_bios_scratch_engine_hung(rdev, true);
4081 /* try soft reset */
4082 si_gpu_soft_reset(rdev, reset_mask);
4084 reset_mask = si_gpu_check_soft_reset(rdev);
4086 /* try pci config reset */
4087 if (reset_mask && radeon_hard_reset)
4088 si_gpu_pci_config_reset(rdev);
4090 reset_mask = si_gpu_check_soft_reset(rdev);
4093 r600_set_bios_scratch_engine_hung(rdev, false);
4099 * si_gfx_is_lockup - Check if the GFX engine is locked up
4101 * @rdev: radeon_device pointer
4102 * @ring: radeon_ring structure holding ring information
4104 * Check if the GFX engine is locked up.
4105 * Returns true if the engine appears to be locked up, false if not.
4107 bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4109 u32 reset_mask = si_gpu_check_soft_reset(rdev);
4111 if (!(reset_mask & (RADEON_RESET_GFX |
4112 RADEON_RESET_COMPUTE |
4113 RADEON_RESET_CP))) {
4114 radeon_ring_lockup_update(rdev, ring);
4117 return radeon_ring_test_lockup(rdev, ring);
4121 static void si_mc_program(struct radeon_device *rdev)
4123 struct evergreen_mc_save save;
4127 /* Initialize HDP */
4128 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
4129 WREG32((0x2c14 + j), 0x00000000);
4130 WREG32((0x2c18 + j), 0x00000000);
4131 WREG32((0x2c1c + j), 0x00000000);
4132 WREG32((0x2c20 + j), 0x00000000);
4133 WREG32((0x2c24 + j), 0x00000000);
4135 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
4137 evergreen_mc_stop(rdev, &save);
4138 if (radeon_mc_wait_for_idle(rdev)) {
4139 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4141 if (!ASIC_IS_NODCE(rdev))
4142 /* Lockout access through VGA aperture*/
4143 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
4144 /* Update configuration */
4145 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
4146 rdev->mc.vram_start >> 12);
4147 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
4148 rdev->mc.vram_end >> 12);
4149 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
4150 rdev->vram_scratch.gpu_addr >> 12);
4151 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
4152 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
4153 WREG32(MC_VM_FB_LOCATION, tmp);
4154 /* XXX double check these! */
4155 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
4156 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
4157 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
4158 WREG32(MC_VM_AGP_BASE, 0);
4159 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
4160 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
4161 if (radeon_mc_wait_for_idle(rdev)) {
4162 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4164 evergreen_mc_resume(rdev, &save);
4165 if (!ASIC_IS_NODCE(rdev)) {
4166 /* we need to own VRAM, so turn off the VGA renderer here
4167 * to stop it overwriting our objects */
4168 rv515_vga_render_disable(rdev);
4172 void si_vram_gtt_location(struct radeon_device *rdev,
4173 struct radeon_mc *mc)
4175 if (mc->mc_vram_size > 0xFFC0000000ULL) {
4176 /* leave room for at least 1024M GTT */
4177 dev_warn(rdev->dev, "limiting VRAM\n");
4178 mc->real_vram_size = 0xFFC0000000ULL;
4179 mc->mc_vram_size = 0xFFC0000000ULL;
4181 radeon_vram_location(rdev, &rdev->mc, 0);
4182 rdev->mc.gtt_base_align = 0;
4183 radeon_gtt_location(rdev, mc);
4186 static int si_mc_init(struct radeon_device *rdev)
4189 int chansize, numchan;
4191 /* Get VRAM informations */
4192 rdev->mc.vram_is_ddr = true;
4193 tmp = RREG32(MC_ARB_RAMCFG);
4194 if (tmp & CHANSIZE_OVERRIDE) {
4196 } else if (tmp & CHANSIZE_MASK) {
4201 tmp = RREG32(MC_SHARED_CHMAP);
4202 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
4232 rdev->mc.vram_width = numchan * chansize;
4233 /* Could aper size report 0 ? */
4234 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
4235 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
4236 /* size in MB on si */
4237 tmp = RREG32(CONFIG_MEMSIZE);
4238 /* some boards may have garbage in the upper 16 bits */
4239 if (tmp & 0xffff0000) {
4240 DRM_INFO("Probable bad vram size: 0x%08x\n", tmp);
4244 rdev->mc.mc_vram_size = tmp * 1024ULL * 1024ULL;
4245 rdev->mc.real_vram_size = rdev->mc.mc_vram_size;
4246 rdev->mc.visible_vram_size = rdev->mc.aper_size;
4247 si_vram_gtt_location(rdev, &rdev->mc);
4248 radeon_update_bandwidth_info(rdev);
4256 void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
4258 /* flush hdp cache */
4259 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
4261 /* bits 0-15 are the VM contexts0-15 */
4262 WREG32(VM_INVALIDATE_REQUEST, 1);
4265 static int si_pcie_gart_enable(struct radeon_device *rdev)
4269 if (rdev->gart.robj == NULL) {
4270 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
4273 r = radeon_gart_table_vram_pin(rdev);
4276 /* Setup TLB control */
4277 WREG32(MC_VM_MX_L1_TLB_CNTL,
4280 ENABLE_L1_FRAGMENT_PROCESSING |
4281 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4282 ENABLE_ADVANCED_DRIVER_MODEL |
4283 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4284 /* Setup L2 cache */
4285 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
4286 ENABLE_L2_FRAGMENT_PROCESSING |
4287 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4288 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4289 EFFECTIVE_L2_QUEUE_SIZE(7) |
4290 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4291 WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
4292 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4294 L2_CACHE_BIGK_FRAGMENT_SIZE(4));
4295 /* setup context0 */
4296 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
4297 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
4298 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
4299 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
4300 (u32)(rdev->dummy_page.addr >> 12));
4301 WREG32(VM_CONTEXT0_CNTL2, 0);
4302 WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
4303 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
4309 /* empty context1-15 */
4310 /* set vm size, must be a multiple of 4 */
4311 WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
4312 WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn - 1);
4313 /* Assign the pt base to something valid for now; the pts used for
4314 * the VMs are determined by the application and setup and assigned
4315 * on the fly in the vm part of radeon_gart.c
4317 for (i = 1; i < 16; i++) {
4319 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
4320 rdev->vm_manager.saved_table_addr[i]);
4322 WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
4323 rdev->vm_manager.saved_table_addr[i]);
4326 /* enable context1-15 */
4327 WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
4328 (u32)(rdev->dummy_page.addr >> 12));
4329 WREG32(VM_CONTEXT1_CNTL2, 4);
4330 WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
4331 PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
4332 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4333 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4334 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4335 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4336 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
4337 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
4338 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
4339 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
4340 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
4341 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
4342 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4343 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
4345 si_pcie_gart_tlb_flush(rdev);
4346 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
4347 (unsigned)(rdev->mc.gtt_size >> 20),
4348 (unsigned long long)rdev->gart.table_addr);
4349 rdev->gart.ready = true;
4353 static void si_pcie_gart_disable(struct radeon_device *rdev)
4357 for (i = 1; i < 16; ++i) {
4360 reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2);
4362 reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2);
4363 rdev->vm_manager.saved_table_addr[i] = RREG32(reg);
4366 /* Disable all tables */
4367 WREG32(VM_CONTEXT0_CNTL, 0);
4368 WREG32(VM_CONTEXT1_CNTL, 0);
4369 /* Setup TLB control */
4370 WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4371 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4372 /* Setup L2 cache */
4373 WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4374 ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4375 EFFECTIVE_L2_QUEUE_SIZE(7) |
4376 CONTEXT1_IDENTITY_ACCESS_MODE(1));
4377 WREG32(VM_L2_CNTL2, 0);
4378 WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4379 L2_CACHE_BIGK_FRAGMENT_SIZE(0));
4380 radeon_gart_table_vram_unpin(rdev);
4383 static void si_pcie_gart_fini(struct radeon_device *rdev)
4385 si_pcie_gart_disable(rdev);
4386 radeon_gart_table_vram_free(rdev);
4387 radeon_gart_fini(rdev);
4391 static bool si_vm_reg_valid(u32 reg)
4393 /* context regs are fine */
4397 /* shader regs are also fine */
4398 if (reg >= 0xB000 && reg < 0xC000)
4401 /* check config regs */
4403 case GRBM_GFX_INDEX:
4404 case CP_STRMOUT_CNTL:
4405 case VGT_VTX_VECT_EJECT_REG:
4406 case VGT_CACHE_INVALIDATION:
4407 case VGT_ESGS_RING_SIZE:
4408 case VGT_GSVS_RING_SIZE:
4409 case VGT_GS_VERTEX_REUSE:
4410 case VGT_PRIMITIVE_TYPE:
4411 case VGT_INDEX_TYPE:
4412 case VGT_NUM_INDICES:
4413 case VGT_NUM_INSTANCES:
4414 case VGT_TF_RING_SIZE:
4415 case VGT_HS_OFFCHIP_PARAM:
4416 case VGT_TF_MEMORY_BASE:
4418 case PA_SU_LINE_STIPPLE_VALUE:
4419 case PA_SC_LINE_STIPPLE_STATE:
4422 case SPI_STATIC_THREAD_MGMT_1:
4423 case SPI_STATIC_THREAD_MGMT_2:
4424 case SPI_STATIC_THREAD_MGMT_3:
4425 case SPI_PS_MAX_WAVE_ID:
4426 case SPI_CONFIG_CNTL:
4427 case SPI_CONFIG_CNTL_1:
4429 case TA_CS_BC_BASE_ADDR:
4432 DRM_ERROR("Invalid register 0x%x in CS\n", reg);
4437 static int si_vm_packet3_ce_check(struct radeon_device *rdev,
4438 u32 *ib, struct radeon_cs_packet *pkt)
4440 switch (pkt->opcode) {
4442 case PACKET3_SET_BASE:
4443 case PACKET3_SET_CE_DE_COUNTERS:
4444 case PACKET3_LOAD_CONST_RAM:
4445 case PACKET3_WRITE_CONST_RAM:
4446 case PACKET3_WRITE_CONST_RAM_OFFSET:
4447 case PACKET3_DUMP_CONST_RAM:
4448 case PACKET3_INCREMENT_CE_COUNTER:
4449 case PACKET3_WAIT_ON_DE_COUNTER:
4450 case PACKET3_CE_WRITE:
4453 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
4459 static int si_vm_packet3_cp_dma_check(u32 *ib, u32 idx)
4461 u32 start_reg, reg, i;
4462 u32 command = ib[idx + 4];
4463 u32 info = ib[idx + 1];
4464 u32 idx_value = ib[idx];
4465 if (command & PACKET3_CP_DMA_CMD_SAS) {
4466 /* src address space is register */
4467 if (((info & 0x60000000) >> 29) == 0) {
4468 start_reg = idx_value << 2;
4469 if (command & PACKET3_CP_DMA_CMD_SAIC) {
4471 if (!si_vm_reg_valid(reg)) {
4472 DRM_ERROR("CP DMA Bad SRC register\n");
4476 for (i = 0; i < (command & 0x1fffff); i++) {
4477 reg = start_reg + (4 * i);
4478 if (!si_vm_reg_valid(reg)) {
4479 DRM_ERROR("CP DMA Bad SRC register\n");
4486 if (command & PACKET3_CP_DMA_CMD_DAS) {
4487 /* dst address space is register */
4488 if (((info & 0x00300000) >> 20) == 0) {
4489 start_reg = ib[idx + 2];
4490 if (command & PACKET3_CP_DMA_CMD_DAIC) {
4492 if (!si_vm_reg_valid(reg)) {
4493 DRM_ERROR("CP DMA Bad DST register\n");
4497 for (i = 0; i < (command & 0x1fffff); i++) {
4498 reg = start_reg + (4 * i);
4499 if (!si_vm_reg_valid(reg)) {
4500 DRM_ERROR("CP DMA Bad DST register\n");
4510 static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
4511 u32 *ib, struct radeon_cs_packet *pkt)
4514 u32 idx = pkt->idx + 1;
4515 u32 idx_value = ib[idx];
4516 u32 start_reg, end_reg, reg, i;
4518 switch (pkt->opcode) {
4520 case PACKET3_SET_BASE:
4521 case PACKET3_CLEAR_STATE:
4522 case PACKET3_INDEX_BUFFER_SIZE:
4523 case PACKET3_DISPATCH_DIRECT:
4524 case PACKET3_DISPATCH_INDIRECT:
4525 case PACKET3_ALLOC_GDS:
4526 case PACKET3_WRITE_GDS_RAM:
4527 case PACKET3_ATOMIC_GDS:
4528 case PACKET3_ATOMIC:
4529 case PACKET3_OCCLUSION_QUERY:
4530 case PACKET3_SET_PREDICATION:
4531 case PACKET3_COND_EXEC:
4532 case PACKET3_PRED_EXEC:
4533 case PACKET3_DRAW_INDIRECT:
4534 case PACKET3_DRAW_INDEX_INDIRECT:
4535 case PACKET3_INDEX_BASE:
4536 case PACKET3_DRAW_INDEX_2:
4537 case PACKET3_CONTEXT_CONTROL:
4538 case PACKET3_INDEX_TYPE:
4539 case PACKET3_DRAW_INDIRECT_MULTI:
4540 case PACKET3_DRAW_INDEX_AUTO:
4541 case PACKET3_DRAW_INDEX_IMMD:
4542 case PACKET3_NUM_INSTANCES:
4543 case PACKET3_DRAW_INDEX_MULTI_AUTO:
4544 case PACKET3_STRMOUT_BUFFER_UPDATE:
4545 case PACKET3_DRAW_INDEX_OFFSET_2:
4546 case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
4547 case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
4548 case PACKET3_MPEG_INDEX:
4549 case PACKET3_WAIT_REG_MEM:
4550 case PACKET3_MEM_WRITE:
4551 case PACKET3_PFP_SYNC_ME:
4552 case PACKET3_SURFACE_SYNC:
4553 case PACKET3_EVENT_WRITE:
4554 case PACKET3_EVENT_WRITE_EOP:
4555 case PACKET3_EVENT_WRITE_EOS:
4556 case PACKET3_SET_CONTEXT_REG:
4557 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4558 case PACKET3_SET_SH_REG:
4559 case PACKET3_SET_SH_REG_OFFSET:
4560 case PACKET3_INCREMENT_DE_COUNTER:
4561 case PACKET3_WAIT_ON_CE_COUNTER:
4562 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4563 case PACKET3_ME_WRITE:
4565 case PACKET3_COPY_DATA:
4566 if ((idx_value & 0xf00) == 0) {
4567 reg = ib[idx + 3] * 4;
4568 if (!si_vm_reg_valid(reg))
4572 case PACKET3_WRITE_DATA:
4573 if ((idx_value & 0xf00) == 0) {
4574 start_reg = ib[idx + 1] * 4;
4575 if (idx_value & 0x10000) {
4576 if (!si_vm_reg_valid(start_reg))
4579 for (i = 0; i < (pkt->count - 2); i++) {
4580 reg = start_reg + (4 * i);
4581 if (!si_vm_reg_valid(reg))
4587 case PACKET3_COND_WRITE:
4588 if (idx_value & 0x100) {
4589 reg = ib[idx + 5] * 4;
4590 if (!si_vm_reg_valid(reg))
4594 case PACKET3_COPY_DW:
4595 if (idx_value & 0x2) {
4596 reg = ib[idx + 3] * 4;
4597 if (!si_vm_reg_valid(reg))
4601 case PACKET3_SET_CONFIG_REG:
4602 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
4603 end_reg = 4 * pkt->count + start_reg - 4;
4604 if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
4605 (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
4606 (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
4607 DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
4610 for (i = 0; i < pkt->count; i++) {
4611 reg = start_reg + (4 * i);
4612 if (!si_vm_reg_valid(reg))
4616 case PACKET3_CP_DMA:
4617 r = si_vm_packet3_cp_dma_check(ib, idx);
4622 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
4628 static int si_vm_packet3_compute_check(struct radeon_device *rdev,
4629 u32 *ib, struct radeon_cs_packet *pkt)
4632 u32 idx = pkt->idx + 1;
4633 u32 idx_value = ib[idx];
4634 u32 start_reg, reg, i;
4636 switch (pkt->opcode) {
4638 case PACKET3_SET_BASE:
4639 case PACKET3_CLEAR_STATE:
4640 case PACKET3_DISPATCH_DIRECT:
4641 case PACKET3_DISPATCH_INDIRECT:
4642 case PACKET3_ALLOC_GDS:
4643 case PACKET3_WRITE_GDS_RAM:
4644 case PACKET3_ATOMIC_GDS:
4645 case PACKET3_ATOMIC:
4646 case PACKET3_OCCLUSION_QUERY:
4647 case PACKET3_SET_PREDICATION:
4648 case PACKET3_COND_EXEC:
4649 case PACKET3_PRED_EXEC:
4650 case PACKET3_CONTEXT_CONTROL:
4651 case PACKET3_STRMOUT_BUFFER_UPDATE:
4652 case PACKET3_WAIT_REG_MEM:
4653 case PACKET3_MEM_WRITE:
4654 case PACKET3_PFP_SYNC_ME:
4655 case PACKET3_SURFACE_SYNC:
4656 case PACKET3_EVENT_WRITE:
4657 case PACKET3_EVENT_WRITE_EOP:
4658 case PACKET3_EVENT_WRITE_EOS:
4659 case PACKET3_SET_CONTEXT_REG:
4660 case PACKET3_SET_CONTEXT_REG_INDIRECT:
4661 case PACKET3_SET_SH_REG:
4662 case PACKET3_SET_SH_REG_OFFSET:
4663 case PACKET3_INCREMENT_DE_COUNTER:
4664 case PACKET3_WAIT_ON_CE_COUNTER:
4665 case PACKET3_WAIT_ON_AVAIL_BUFFER:
4666 case PACKET3_ME_WRITE:
4668 case PACKET3_COPY_DATA:
4669 if ((idx_value & 0xf00) == 0) {
4670 reg = ib[idx + 3] * 4;
4671 if (!si_vm_reg_valid(reg))
4675 case PACKET3_WRITE_DATA:
4676 if ((idx_value & 0xf00) == 0) {
4677 start_reg = ib[idx + 1] * 4;
4678 if (idx_value & 0x10000) {
4679 if (!si_vm_reg_valid(start_reg))
4682 for (i = 0; i < (pkt->count - 2); i++) {
4683 reg = start_reg + (4 * i);
4684 if (!si_vm_reg_valid(reg))
4690 case PACKET3_COND_WRITE:
4691 if (idx_value & 0x100) {
4692 reg = ib[idx + 5] * 4;
4693 if (!si_vm_reg_valid(reg))
4697 case PACKET3_COPY_DW:
4698 if (idx_value & 0x2) {
4699 reg = ib[idx + 3] * 4;
4700 if (!si_vm_reg_valid(reg))
4704 case PACKET3_CP_DMA:
4705 r = si_vm_packet3_cp_dma_check(ib, idx);
4710 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
4716 int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4720 struct radeon_cs_packet pkt;
4724 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
4725 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
4728 case RADEON_PACKET_TYPE0:
4729 dev_err(rdev->dev, "Packet0 not allowed!\n");
4732 case RADEON_PACKET_TYPE2:
4735 case RADEON_PACKET_TYPE3:
4736 pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
4737 if (ib->is_const_ib)
4738 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
4741 case RADEON_RING_TYPE_GFX_INDEX:
4742 ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
4744 case CAYMAN_RING_TYPE_CP1_INDEX:
4745 case CAYMAN_RING_TYPE_CP2_INDEX:
4746 ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
4749 dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
4754 idx += pkt.count + 2;
4757 dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
4762 for (i = 0; i < ib->length_dw; i++) {
4764 printk("\t0x%08x <---\n", ib->ptr[i]);
4766 printk("\t0x%08x\n", ib->ptr[i]);
4770 } while (idx < ib->length_dw);
4778 int si_vm_init(struct radeon_device *rdev)
4781 rdev->vm_manager.nvm = 16;
4782 /* base offset of vram pages */
4783 rdev->vm_manager.vram_base_offset = 0;
4788 void si_vm_fini(struct radeon_device *rdev)
4793 * si_vm_decode_fault - print human readable fault info
4795 * @rdev: radeon_device pointer
4796 * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
4797 * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
4799 * Print human readable fault information (SI).
4801 static void si_vm_decode_fault(struct radeon_device *rdev,
4802 u32 status, u32 addr)
4804 u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
4805 u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
4806 u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
4809 if (rdev->family == CHIP_TAHITI) {
5050 printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
5051 protections, vmid, addr,
5052 (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
5056 void si_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
5057 unsigned vm_id, uint64_t pd_addr)
5059 /* write new base address */
5060 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5061 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5062 WRITE_DATA_DST_SEL(0)));
5065 radeon_ring_write(ring,
5066 (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2);
5068 radeon_ring_write(ring,
5069 (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2);
5071 radeon_ring_write(ring, 0);
5072 radeon_ring_write(ring, pd_addr >> 12);
5074 /* flush hdp cache */
5075 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5076 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5077 WRITE_DATA_DST_SEL(0)));
5078 radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
5079 radeon_ring_write(ring, 0);
5080 radeon_ring_write(ring, 0x1);
5082 /* bits 0-15 are the VM contexts0-15 */
5083 radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5084 radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5085 WRITE_DATA_DST_SEL(0)));
5086 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5087 radeon_ring_write(ring, 0);
5088 radeon_ring_write(ring, 1 << vm_id);
5090 /* wait for the invalidate to complete */
5091 radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
5092 radeon_ring_write(ring, (WAIT_REG_MEM_FUNCTION(0) | /* always */
5093 WAIT_REG_MEM_ENGINE(0))); /* me */
5094 radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5095 radeon_ring_write(ring, 0);
5096 radeon_ring_write(ring, 0); /* ref */
5097 radeon_ring_write(ring, 0); /* mask */
5098 radeon_ring_write(ring, 0x20); /* poll interval */
5100 /* sync PFP to ME, otherwise we might get invalid PFP reads */
5101 radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5102 radeon_ring_write(ring, 0x0);
5106 * Power and clock gating
5108 static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
5112 for (i = 0; i < rdev->usec_timeout; i++) {
5113 if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
5118 for (i = 0; i < rdev->usec_timeout; i++) {
5119 if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
5125 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
5128 u32 tmp = RREG32(CP_INT_CNTL_RING0);
5133 tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5135 tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5136 WREG32(CP_INT_CNTL_RING0, tmp);
5139 /* read a gfx register */
5140 tmp = RREG32(DB_DEPTH_INFO);
5142 mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS;
5143 for (i = 0; i < rdev->usec_timeout; i++) {
5144 if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
5151 static void si_set_uvd_dcm(struct radeon_device *rdev,
5156 tmp = RREG32(UVD_CGC_CTRL);
5157 tmp &= ~(CLK_OD_MASK | CG_DT_MASK);
5158 tmp |= DCM | CG_DT(1) | CLK_OD(4);
5162 tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7);
5168 WREG32(UVD_CGC_CTRL, tmp);
5169 WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2);
5172 void si_init_uvd_internal_cg(struct radeon_device *rdev)
5174 bool hw_mode = true;
5177 si_set_uvd_dcm(rdev, false);
5179 u32 tmp = RREG32(UVD_CGC_CTRL);
5181 WREG32(UVD_CGC_CTRL, tmp);
5185 static u32 si_halt_rlc(struct radeon_device *rdev)
5189 orig = data = RREG32(RLC_CNTL);
5191 if (data & RLC_ENABLE) {
5192 data &= ~RLC_ENABLE;
5193 WREG32(RLC_CNTL, data);
5195 si_wait_for_rlc_serdes(rdev);
5201 static void si_update_rlc(struct radeon_device *rdev, u32 rlc)
5205 tmp = RREG32(RLC_CNTL);
5207 WREG32(RLC_CNTL, rlc);
5210 static void si_enable_dma_pg(struct radeon_device *rdev, bool enable)
5214 orig = data = RREG32(DMA_PG);
5215 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA))
5216 data |= PG_CNTL_ENABLE;
5218 data &= ~PG_CNTL_ENABLE;
5220 WREG32(DMA_PG, data);
5223 static void si_init_dma_pg(struct radeon_device *rdev)
5227 WREG32(DMA_PGFSM_WRITE, 0x00002000);
5228 WREG32(DMA_PGFSM_CONFIG, 0x100010ff);
5230 for (tmp = 0; tmp < 5; tmp++)
5231 WREG32(DMA_PGFSM_WRITE, 0);
5234 static void si_enable_gfx_cgpg(struct radeon_device *rdev,
5239 if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
5240 tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10);
5241 WREG32(RLC_TTOP_D, tmp);
5243 tmp = RREG32(RLC_PG_CNTL);
5244 tmp |= GFX_PG_ENABLE;
5245 WREG32(RLC_PG_CNTL, tmp);
5247 tmp = RREG32(RLC_AUTO_PG_CTRL);
5249 WREG32(RLC_AUTO_PG_CTRL, tmp);
5251 tmp = RREG32(RLC_AUTO_PG_CTRL);
5253 WREG32(RLC_AUTO_PG_CTRL, tmp);
5255 tmp = RREG32(DB_RENDER_CONTROL);
5259 static void si_init_gfx_cgpg(struct radeon_device *rdev)
5263 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5265 tmp = RREG32(RLC_PG_CNTL);
5267 WREG32(RLC_PG_CNTL, tmp);
5269 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5271 tmp = RREG32(RLC_AUTO_PG_CTRL);
5273 tmp &= ~GRBM_REG_SGIT_MASK;
5274 tmp |= GRBM_REG_SGIT(0x700);
5275 tmp &= ~PG_AFTER_GRBM_REG_ST_MASK;
5276 WREG32(RLC_AUTO_PG_CTRL, tmp);
5279 static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
5281 u32 mask = 0, tmp, tmp1;
5284 si_select_se_sh(rdev, se, sh);
5285 tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
5286 tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
5287 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5294 for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) {
5299 return (~tmp) & mask;
5302 static void si_init_ao_cu_mask(struct radeon_device *rdev)
5304 u32 i, j, k, active_cu_number = 0;
5305 u32 mask, counter, cu_bitmap;
5308 for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
5309 for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
5313 for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) {
5314 if (si_get_cu_active_bitmap(rdev, i, j) & mask) {
5322 active_cu_number += counter;
5323 tmp |= (cu_bitmap << (i * 16 + j * 8));
5327 WREG32(RLC_PG_AO_CU_MASK, tmp);
5329 tmp = RREG32(RLC_MAX_PG_CU);
5330 tmp &= ~MAX_PU_CU_MASK;
5331 tmp |= MAX_PU_CU(active_cu_number);
5332 WREG32(RLC_MAX_PG_CU, tmp);
5335 static void si_enable_cgcg(struct radeon_device *rdev,
5338 u32 data, orig, tmp;
5340 orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5342 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5343 si_enable_gui_idle_interrupt(rdev, true);
5345 WREG32(RLC_GCPM_GENERAL_3, 0x00000080);
5347 tmp = si_halt_rlc(rdev);
5349 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5350 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5351 WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff);
5353 si_wait_for_rlc_serdes(rdev);
5355 si_update_rlc(rdev, tmp);
5357 WREG32(RLC_SERDES_WR_CTRL, 0x007000ff);
5359 data |= CGCG_EN | CGLS_EN;
5361 si_enable_gui_idle_interrupt(rdev, false);
5363 RREG32(CB_CGTT_SCLK_CTRL);
5364 RREG32(CB_CGTT_SCLK_CTRL);
5365 RREG32(CB_CGTT_SCLK_CTRL);
5366 RREG32(CB_CGTT_SCLK_CTRL);
5368 data &= ~(CGCG_EN | CGLS_EN);
5372 WREG32(RLC_CGCG_CGLS_CTRL, data);
5375 static void si_enable_mgcg(struct radeon_device *rdev,
5378 u32 data, orig, tmp = 0;
5380 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5381 orig = data = RREG32(CGTS_SM_CTRL_REG);
5384 WREG32(CGTS_SM_CTRL_REG, data);
5386 if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5387 orig = data = RREG32(CP_MEM_SLP_CNTL);
5388 data |= CP_MEM_LS_EN;
5390 WREG32(CP_MEM_SLP_CNTL, data);
5393 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5396 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5398 tmp = si_halt_rlc(rdev);
5400 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5401 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5402 WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff);
5404 si_update_rlc(rdev, tmp);
5406 orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5409 WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5411 data = RREG32(CP_MEM_SLP_CNTL);
5412 if (data & CP_MEM_LS_EN) {
5413 data &= ~CP_MEM_LS_EN;
5414 WREG32(CP_MEM_SLP_CNTL, data);
5416 orig = data = RREG32(CGTS_SM_CTRL_REG);
5417 data |= LS_OVERRIDE | OVERRIDE;
5419 WREG32(CGTS_SM_CTRL_REG, data);
5421 tmp = si_halt_rlc(rdev);
5423 WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5424 WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5425 WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff);
5427 si_update_rlc(rdev, tmp);
5431 static void si_enable_uvd_mgcg(struct radeon_device *rdev,
5434 u32 orig, data, tmp;
5436 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
5437 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5439 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5441 orig = data = RREG32(UVD_CGC_CTRL);
5444 WREG32(UVD_CGC_CTRL, data);
5446 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0);
5447 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0);
5449 tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5451 WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5453 orig = data = RREG32(UVD_CGC_CTRL);
5456 WREG32(UVD_CGC_CTRL, data);
5458 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff);
5459 WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff);
5463 static const u32 mc_cg_registers[] =
5476 static void si_enable_mc_ls(struct radeon_device *rdev,
5482 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5483 orig = data = RREG32(mc_cg_registers[i]);
5484 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
5485 data |= MC_LS_ENABLE;
5487 data &= ~MC_LS_ENABLE;
5489 WREG32(mc_cg_registers[i], data);
5493 static void si_enable_mc_mgcg(struct radeon_device *rdev,
5499 for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5500 orig = data = RREG32(mc_cg_registers[i]);
5501 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
5502 data |= MC_CG_ENABLE;
5504 data &= ~MC_CG_ENABLE;
5506 WREG32(mc_cg_registers[i], data);
5510 static void si_enable_dma_mgcg(struct radeon_device *rdev,
5513 u32 orig, data, offset;
5516 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
5517 for (i = 0; i < 2; i++) {
5519 offset = DMA0_REGISTER_OFFSET;
5521 offset = DMA1_REGISTER_OFFSET;
5522 orig = data = RREG32(DMA_POWER_CNTL + offset);
5523 data &= ~MEM_POWER_OVERRIDE;
5525 WREG32(DMA_POWER_CNTL + offset, data);
5526 WREG32(DMA_CLK_CTRL + offset, 0x00000100);
5529 for (i = 0; i < 2; i++) {
5531 offset = DMA0_REGISTER_OFFSET;
5533 offset = DMA1_REGISTER_OFFSET;
5534 orig = data = RREG32(DMA_POWER_CNTL + offset);
5535 data |= MEM_POWER_OVERRIDE;
5537 WREG32(DMA_POWER_CNTL + offset, data);
5539 orig = data = RREG32(DMA_CLK_CTRL + offset);
5542 WREG32(DMA_CLK_CTRL + offset, data);
5547 static void si_enable_bif_mgls(struct radeon_device *rdev,
5552 orig = data = RREG32_PCIE(PCIE_CNTL2);
5554 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5555 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5556 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5558 data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5559 REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5562 WREG32_PCIE(PCIE_CNTL2, data);
5565 static void si_enable_hdp_mgcg(struct radeon_device *rdev,
5570 orig = data = RREG32(HDP_HOST_PATH_CNTL);
5572 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
5573 data &= ~CLOCK_GATING_DIS;
5575 data |= CLOCK_GATING_DIS;
5578 WREG32(HDP_HOST_PATH_CNTL, data);
5581 static void si_enable_hdp_ls(struct radeon_device *rdev,
5586 orig = data = RREG32(HDP_MEM_POWER_LS);
5588 if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
5589 data |= HDP_LS_ENABLE;
5591 data &= ~HDP_LS_ENABLE;
5594 WREG32(HDP_MEM_POWER_LS, data);
5597 static void si_update_cg(struct radeon_device *rdev,
5598 u32 block, bool enable)
5600 if (block & RADEON_CG_BLOCK_GFX) {
5601 si_enable_gui_idle_interrupt(rdev, false);
5602 /* order matters! */
5604 si_enable_mgcg(rdev, true);
5605 si_enable_cgcg(rdev, true);
5607 si_enable_cgcg(rdev, false);
5608 si_enable_mgcg(rdev, false);
5610 si_enable_gui_idle_interrupt(rdev, true);
5613 if (block & RADEON_CG_BLOCK_MC) {
5614 si_enable_mc_mgcg(rdev, enable);
5615 si_enable_mc_ls(rdev, enable);
5618 if (block & RADEON_CG_BLOCK_SDMA) {
5619 si_enable_dma_mgcg(rdev, enable);
5622 if (block & RADEON_CG_BLOCK_BIF) {
5623 si_enable_bif_mgls(rdev, enable);
5626 if (block & RADEON_CG_BLOCK_UVD) {
5627 if (rdev->has_uvd) {
5628 si_enable_uvd_mgcg(rdev, enable);
5632 if (block & RADEON_CG_BLOCK_HDP) {
5633 si_enable_hdp_mgcg(rdev, enable);
5634 si_enable_hdp_ls(rdev, enable);
5638 static void si_init_cg(struct radeon_device *rdev)
5640 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5641 RADEON_CG_BLOCK_MC |
5642 RADEON_CG_BLOCK_SDMA |
5643 RADEON_CG_BLOCK_BIF |
5644 RADEON_CG_BLOCK_HDP), true);
5645 if (rdev->has_uvd) {
5646 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, true);
5647 si_init_uvd_internal_cg(rdev);
5651 static void si_fini_cg(struct radeon_device *rdev)
5653 if (rdev->has_uvd) {
5654 si_update_cg(rdev, RADEON_CG_BLOCK_UVD, false);
5656 si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5657 RADEON_CG_BLOCK_MC |
5658 RADEON_CG_BLOCK_SDMA |
5659 RADEON_CG_BLOCK_BIF |
5660 RADEON_CG_BLOCK_HDP), false);
5663 u32 si_get_csb_size(struct radeon_device *rdev)
5666 const struct cs_section_def *sect = NULL;
5667 const struct cs_extent_def *ext = NULL;
5669 if (rdev->rlc.cs_data == NULL)
5672 /* begin clear state */
5674 /* context control state */
5677 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5678 for (ext = sect->section; ext->extent != NULL; ++ext) {
5679 if (sect->id == SECT_CONTEXT)
5680 count += 2 + ext->reg_count;
5685 /* pa_sc_raster_config */
5687 /* end clear state */
5695 void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
5698 const struct cs_section_def *sect = NULL;
5699 const struct cs_extent_def *ext = NULL;
5701 if (rdev->rlc.cs_data == NULL)
5706 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5707 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
5709 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5710 buffer[count++] = cpu_to_le32(0x80000000);
5711 buffer[count++] = cpu_to_le32(0x80000000);
5713 for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5714 for (ext = sect->section; ext->extent != NULL; ++ext) {
5715 if (sect->id == SECT_CONTEXT) {
5717 cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
5718 buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
5719 for (i = 0; i < ext->reg_count; i++)
5720 buffer[count++] = cpu_to_le32(ext->extent[i]);
5727 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
5728 buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
5729 switch (rdev->family) {
5732 buffer[count++] = cpu_to_le32(0x2a00126a);
5735 buffer[count++] = cpu_to_le32(0x0000124a);
5738 buffer[count++] = cpu_to_le32(0x00000082);
5741 buffer[count++] = cpu_to_le32(0x00000000);
5744 buffer[count++] = cpu_to_le32(0x00000000);
5748 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5749 buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
5751 buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
5752 buffer[count++] = cpu_to_le32(0);
5755 static void si_init_pg(struct radeon_device *rdev)
5757 if (rdev->pg_flags) {
5758 if (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA) {
5759 si_init_dma_pg(rdev);
5761 si_init_ao_cu_mask(rdev);
5762 if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
5763 si_init_gfx_cgpg(rdev);
5765 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5766 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5768 si_enable_dma_pg(rdev, true);
5769 si_enable_gfx_cgpg(rdev, true);
5771 WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5772 WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5776 static void si_fini_pg(struct radeon_device *rdev)
5778 if (rdev->pg_flags) {
5779 si_enable_dma_pg(rdev, false);
5780 si_enable_gfx_cgpg(rdev, false);
5787 void si_rlc_reset(struct radeon_device *rdev)
5789 u32 tmp = RREG32(GRBM_SOFT_RESET);
5791 tmp |= SOFT_RESET_RLC;
5792 WREG32(GRBM_SOFT_RESET, tmp);
5794 tmp &= ~SOFT_RESET_RLC;
5795 WREG32(GRBM_SOFT_RESET, tmp);
5799 static void si_rlc_stop(struct radeon_device *rdev)
5801 WREG32(RLC_CNTL, 0);
5803 si_enable_gui_idle_interrupt(rdev, false);
5805 si_wait_for_rlc_serdes(rdev);
5808 static void si_rlc_start(struct radeon_device *rdev)
5810 WREG32(RLC_CNTL, RLC_ENABLE);
5812 si_enable_gui_idle_interrupt(rdev, true);
5817 static bool si_lbpw_supported(struct radeon_device *rdev)
5821 /* Enable LBPW only for DDR3 */
5822 tmp = RREG32(MC_SEQ_MISC0);
5823 if ((tmp & 0xF0000000) == 0xB0000000)
5828 static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
5832 tmp = RREG32(RLC_LB_CNTL);
5834 tmp |= LOAD_BALANCE_ENABLE;
5836 tmp &= ~LOAD_BALANCE_ENABLE;
5837 WREG32(RLC_LB_CNTL, tmp);
5840 si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5841 WREG32(SPI_LB_CU_MASK, 0x00ff);
5845 static int si_rlc_resume(struct radeon_device *rdev)
5860 WREG32(RLC_RL_BASE, 0);
5861 WREG32(RLC_RL_SIZE, 0);
5862 WREG32(RLC_LB_CNTL, 0);
5863 WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
5864 WREG32(RLC_LB_CNTR_INIT, 0);
5865 WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5867 WREG32(RLC_MC_CNTL, 0);
5868 WREG32(RLC_UCODE_CNTL, 0);
5871 const struct rlc_firmware_header_v1_0 *hdr =
5872 (const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
5873 u32 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
5874 const __le32 *fw_data = (const __le32 *)
5875 (rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
5877 radeon_ucode_print_rlc_hdr(&hdr->header);
5879 for (i = 0; i < fw_size; i++) {
5880 WREG32(RLC_UCODE_ADDR, i);
5881 WREG32(RLC_UCODE_DATA, le32_to_cpup(fw_data++));
5884 const __be32 *fw_data =
5885 (const __be32 *)rdev->rlc_fw->data;
5886 for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
5887 WREG32(RLC_UCODE_ADDR, i);
5888 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
5891 WREG32(RLC_UCODE_ADDR, 0);
5893 si_enable_lbpw(rdev, si_lbpw_supported(rdev));
5900 static void si_enable_interrupts(struct radeon_device *rdev)
5902 u32 ih_cntl = RREG32(IH_CNTL);
5903 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5905 ih_cntl |= ENABLE_INTR;
5906 ih_rb_cntl |= IH_RB_ENABLE;
5907 WREG32(IH_CNTL, ih_cntl);
5908 WREG32(IH_RB_CNTL, ih_rb_cntl);
5909 rdev->ih.enabled = true;
5912 static void si_disable_interrupts(struct radeon_device *rdev)
5914 u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5915 u32 ih_cntl = RREG32(IH_CNTL);
5917 ih_rb_cntl &= ~IH_RB_ENABLE;
5918 ih_cntl &= ~ENABLE_INTR;
5919 WREG32(IH_RB_CNTL, ih_rb_cntl);
5920 WREG32(IH_CNTL, ih_cntl);
5921 /* set rptr, wptr to 0 */
5922 WREG32(IH_RB_RPTR, 0);
5923 WREG32(IH_RB_WPTR, 0);
5924 rdev->ih.enabled = false;
5928 static void si_disable_interrupt_state(struct radeon_device *rdev)
5933 tmp = RREG32(CP_INT_CNTL_RING0) &
5934 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5935 WREG32(CP_INT_CNTL_RING0, tmp);
5936 WREG32(CP_INT_CNTL_RING1, 0);
5937 WREG32(CP_INT_CNTL_RING2, 0);
5938 tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5939 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
5940 tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5941 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
5942 WREG32(GRBM_INT_CNTL, 0);
5943 WREG32(SRBM_INT_CNTL, 0);
5944 for (i = 0; i < rdev->num_crtc; i++)
5945 WREG32(INT_MASK + crtc_offsets[i], 0);
5946 for (i = 0; i < rdev->num_crtc; i++)
5947 WREG32(GRPH_INT_CONTROL + crtc_offsets[i], 0);
5949 if (!ASIC_IS_NODCE(rdev)) {
5950 WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
5952 for (i = 0; i < 6; i++)
5953 WREG32_AND(DC_HPDx_INT_CONTROL(i),
5954 DC_HPDx_INT_POLARITY);
5958 static int si_irq_init(struct radeon_device *rdev)
5962 u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
5965 ret = r600_ih_ring_alloc(rdev);
5970 si_disable_interrupts(rdev);
5973 ret = si_rlc_resume(rdev);
5975 r600_ih_ring_fini(rdev);
5979 /* setup interrupt control */
5980 /* set dummy read address to dummy page address */
5981 WREG32(INTERRUPT_CNTL2, rdev->dummy_page.addr >> 8);
5982 interrupt_cntl = RREG32(INTERRUPT_CNTL);
5983 /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
5984 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
5986 interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
5987 /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
5988 interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
5989 WREG32(INTERRUPT_CNTL, interrupt_cntl);
5991 WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
5992 rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
5994 ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
5995 IH_WPTR_OVERFLOW_CLEAR |
5998 if (rdev->wb.enabled)
5999 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
6001 /* set the writeback address whether it's enabled or not */
6002 WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
6003 WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6005 WREG32(IH_RB_CNTL, ih_rb_cntl);
6007 /* set rptr, wptr to 0 */
6008 WREG32(IH_RB_RPTR, 0);
6009 WREG32(IH_RB_WPTR, 0);
6011 /* Default settings for IH_CNTL (disabled at first) */
6012 ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6013 /* RPTR_REARM only works if msi's are enabled */
6014 if (rdev->msi_enabled)
6015 ih_cntl |= RPTR_REARM;
6016 WREG32(IH_CNTL, ih_cntl);
6018 /* force the active interrupt state to all disabled */
6019 si_disable_interrupt_state(rdev);
6021 pci_set_master(rdev->pdev);
6024 si_enable_interrupts(rdev);
6029 /* The order we write back each register here is important */
6030 int si_irq_set(struct radeon_device *rdev)
6034 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
6035 u32 grbm_int_cntl = 0;
6036 u32 dma_cntl, dma_cntl1;
6037 u32 thermal_int = 0;
6039 if (!rdev->irq.installed) {
6040 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6043 /* don't enable anything if the ih is disabled */
6044 if (!rdev->ih.enabled) {
6045 si_disable_interrupts(rdev);
6046 /* force the active interrupt state to all disabled */
6047 si_disable_interrupt_state(rdev);
6051 cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6052 (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6054 dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6055 dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6057 thermal_int = RREG32(CG_THERMAL_INT) &
6058 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
6060 /* enable CP interrupts on all rings */
6061 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6062 DRM_DEBUG("si_irq_set: sw int gfx\n");
6063 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6065 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6066 DRM_DEBUG("si_irq_set: sw int cp1\n");
6067 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
6069 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
6070 DRM_DEBUG("si_irq_set: sw int cp2\n");
6071 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
6073 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
6074 DRM_DEBUG("si_irq_set: sw int dma\n");
6075 dma_cntl |= TRAP_ENABLE;
6078 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
6079 DRM_DEBUG("si_irq_set: sw int dma1\n");
6080 dma_cntl1 |= TRAP_ENABLE;
6083 WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
6084 WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
6085 WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
6087 WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
6088 WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
6090 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
6092 if (rdev->irq.dpm_thermal) {
6093 DRM_DEBUG("dpm thermal\n");
6094 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
6097 for (i = 0; i < rdev->num_crtc; i++) {
6098 radeon_irq_kms_set_irq_n_enabled(
6099 rdev, INT_MASK + crtc_offsets[i], VBLANK_INT_MASK,
6100 rdev->irq.crtc_vblank_int[i] ||
6101 atomic_read(&rdev->irq.pflip[i]), "vblank", i);
6104 for (i = 0; i < rdev->num_crtc; i++)
6105 WREG32(GRPH_INT_CONTROL + crtc_offsets[i], GRPH_PFLIP_INT_MASK);
6107 if (!ASIC_IS_NODCE(rdev)) {
6108 for (i = 0; i < 6; i++) {
6109 radeon_irq_kms_set_irq_n_enabled(
6110 rdev, DC_HPDx_INT_CONTROL(i),
6111 DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN,
6112 rdev->irq.hpd[i], "HPD", i);
6116 WREG32(CG_THERMAL_INT, thermal_int);
6119 RREG32(SRBM_STATUS);
6124 /* The order we write back each register here is important */
6125 static inline void si_irq_ack(struct radeon_device *rdev)
6128 u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
6129 u32 *grph_int = rdev->irq.stat_regs.evergreen.grph_int;
6131 if (ASIC_IS_NODCE(rdev))
6134 for (i = 0; i < 6; i++) {
6135 disp_int[i] = RREG32(si_disp_int_status[i]);
6136 if (i < rdev->num_crtc)
6137 grph_int[i] = RREG32(GRPH_INT_STATUS + crtc_offsets[i]);
6140 /* We write back each interrupt register in pairs of two */
6141 for (i = 0; i < rdev->num_crtc; i += 2) {
6142 for (j = i; j < (i + 2); j++) {
6143 if (grph_int[j] & GRPH_PFLIP_INT_OCCURRED)
6144 WREG32(GRPH_INT_STATUS + crtc_offsets[j],
6145 GRPH_PFLIP_INT_CLEAR);
6148 for (j = i; j < (i + 2); j++) {
6149 if (disp_int[j] & LB_D1_VBLANK_INTERRUPT)
6150 WREG32(VBLANK_STATUS + crtc_offsets[j],
6152 if (disp_int[j] & LB_D1_VLINE_INTERRUPT)
6153 WREG32(VLINE_STATUS + crtc_offsets[j],
6158 for (i = 0; i < 6; i++) {
6159 if (disp_int[i] & DC_HPD1_INTERRUPT)
6160 WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_ACK);
6163 for (i = 0; i < 6; i++) {
6164 if (disp_int[i] & DC_HPD1_RX_INTERRUPT)
6165 WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_RX_INT_ACK);
6169 static void si_irq_disable(struct radeon_device *rdev)
6171 si_disable_interrupts(rdev);
6172 /* Wait and acknowledge irq */
6175 si_disable_interrupt_state(rdev);
6178 static void si_irq_suspend(struct radeon_device *rdev)
6180 si_irq_disable(rdev);
6184 static void si_irq_fini(struct radeon_device *rdev)
6186 si_irq_suspend(rdev);
6187 r600_ih_ring_fini(rdev);
6190 static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
6194 if (rdev->wb.enabled)
6195 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6197 wptr = RREG32(IH_RB_WPTR);
6199 if (wptr & RB_OVERFLOW) {
6200 wptr &= ~RB_OVERFLOW;
6201 /* When a ring buffer overflow happen start parsing interrupt
6202 * from the last not overwritten vector (wptr + 16). Hopefully
6203 * this should allow us to catchup.
6205 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
6206 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
6207 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6208 tmp = RREG32(IH_RB_CNTL);
6209 tmp |= IH_WPTR_OVERFLOW_CLEAR;
6210 WREG32(IH_RB_CNTL, tmp);
6212 return (wptr & rdev->ih.ptr_mask);
6216 * Each IV ring entry is 128 bits:
6217 * [7:0] - interrupt source id
6219 * [59:32] - interrupt source data
6220 * [63:60] - reserved
6223 * [127:80] - reserved
6225 int si_irq_process(struct radeon_device *rdev)
6227 u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
6228 u32 crtc_idx, hpd_idx;
6232 u32 src_id, src_data, ring_id;
6234 bool queue_hotplug = false;
6235 bool queue_dp = false;
6236 bool queue_thermal = false;
6238 const char *event_name;
6240 if (!rdev->ih.enabled || rdev->shutdown)
6243 wptr = si_get_ih_wptr(rdev);
6246 /* is somebody else already processing irqs? */
6247 if (atomic_xchg(&rdev->ih.lock, 1))
6250 rptr = rdev->ih.rptr;
6251 DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
6253 /* Order reading of wptr vs. reading of IH ring data */
6256 /* display interrupts */
6259 while (rptr != wptr) {
6260 /* wptr/rptr are in bytes! */
6261 ring_index = rptr / 4;
6262 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
6263 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
6264 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
6267 case 1: /* D1 vblank/vline */
6268 case 2: /* D2 vblank/vline */
6269 case 3: /* D3 vblank/vline */
6270 case 4: /* D4 vblank/vline */
6271 case 5: /* D5 vblank/vline */
6272 case 6: /* D6 vblank/vline */
6273 crtc_idx = src_id - 1;
6275 if (src_data == 0) { /* vblank */
6276 mask = LB_D1_VBLANK_INTERRUPT;
6277 event_name = "vblank";
6279 if (rdev->irq.crtc_vblank_int[crtc_idx]) {
6280 drm_handle_vblank(rdev_to_drm(rdev), crtc_idx);
6281 rdev->pm.vblank_sync = true;
6282 wake_up(&rdev->irq.vblank_queue);
6284 if (atomic_read(&rdev->irq.pflip[crtc_idx])) {
6285 radeon_crtc_handle_vblank(rdev,
6289 } else if (src_data == 1) { /* vline */
6290 mask = LB_D1_VLINE_INTERRUPT;
6291 event_name = "vline";
6293 DRM_DEBUG("Unhandled interrupt: %d %d\n",
6298 if (!(disp_int[crtc_idx] & mask)) {
6299 DRM_DEBUG("IH: D%d %s - IH event w/o asserted irq bit?\n",
6300 crtc_idx + 1, event_name);
6303 disp_int[crtc_idx] &= ~mask;
6304 DRM_DEBUG("IH: D%d %s\n", crtc_idx + 1, event_name);
6307 case 8: /* D1 page flip */
6308 case 10: /* D2 page flip */
6309 case 12: /* D3 page flip */
6310 case 14: /* D4 page flip */
6311 case 16: /* D5 page flip */
6312 case 18: /* D6 page flip */
6313 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
6314 if (radeon_use_pflipirq > 0)
6315 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
6317 case 42: /* HPD hotplug */
6318 if (src_data <= 5) {
6320 mask = DC_HPD1_INTERRUPT;
6321 queue_hotplug = true;
6324 } else if (src_data <= 11) {
6325 hpd_idx = src_data - 6;
6326 mask = DC_HPD1_RX_INTERRUPT;
6328 event_name = "HPD_RX";
6331 DRM_DEBUG("Unhandled interrupt: %d %d\n",
6336 if (!(disp_int[hpd_idx] & mask))
6337 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6339 disp_int[hpd_idx] &= ~mask;
6340 DRM_DEBUG("IH: %s%d\n", event_name, hpd_idx + 1);
6343 DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
6344 WREG32(SRBM_INT_ACK, 0x1);
6347 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
6348 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
6352 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
6353 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
6354 /* reset addr and status */
6355 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
6356 if (addr == 0x0 && status == 0x0)
6358 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
6359 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
6361 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
6363 si_vm_decode_fault(rdev, status, addr);
6365 case 176: /* RINGID0 CP_INT */
6366 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6368 case 177: /* RINGID1 CP_INT */
6369 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6371 case 178: /* RINGID2 CP_INT */
6372 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6374 case 181: /* CP EOP event */
6375 DRM_DEBUG("IH: CP EOP\n");
6378 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6381 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6384 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6388 case 224: /* DMA trap event */
6389 DRM_DEBUG("IH: DMA trap\n");
6390 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6392 case 230: /* thermal low to high */
6393 DRM_DEBUG("IH: thermal low to high\n");
6394 rdev->pm.dpm.thermal.high_to_low = false;
6395 queue_thermal = true;
6397 case 231: /* thermal high to low */
6398 DRM_DEBUG("IH: thermal high to low\n");
6399 rdev->pm.dpm.thermal.high_to_low = true;
6400 queue_thermal = true;
6402 case 233: /* GUI IDLE */
6403 DRM_DEBUG("IH: GUI idle\n");
6405 case 244: /* DMA trap event */
6406 DRM_DEBUG("IH: DMA1 trap\n");
6407 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6410 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6414 /* wptr/rptr are in bytes! */
6416 rptr &= rdev->ih.ptr_mask;
6417 WREG32(IH_RB_RPTR, rptr);
6420 schedule_work(&rdev->dp_work);
6422 schedule_delayed_work(&rdev->hotplug_work, 0);
6423 if (queue_thermal && rdev->pm.dpm_enabled)
6424 schedule_work(&rdev->pm.dpm.thermal.work);
6425 rdev->ih.rptr = rptr;
6426 atomic_set(&rdev->ih.lock, 0);
6428 /* make sure wptr hasn't changed while processing */
6429 wptr = si_get_ih_wptr(rdev);
6437 * startup/shutdown callbacks
6439 static void si_uvd_init(struct radeon_device *rdev)
6446 r = radeon_uvd_init(rdev);
6448 dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
6450 * At this point rdev->uvd.vcpu_bo is NULL which trickles down
6451 * to early fails uvd_v2_2_resume() and thus nothing happens
6452 * there. So it is pointless to try to go through that code
6453 * hence why we disable uvd here.
6455 rdev->has_uvd = false;
6458 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
6459 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
6462 static void si_uvd_start(struct radeon_device *rdev)
6469 r = uvd_v2_2_resume(rdev);
6471 dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
6474 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
6476 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
6482 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
6485 static void si_uvd_resume(struct radeon_device *rdev)
6487 struct radeon_ring *ring;
6490 if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
6493 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
6494 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
6496 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
6499 r = uvd_v1_0_init(rdev);
6501 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
6506 static void si_vce_init(struct radeon_device *rdev)
6513 r = radeon_vce_init(rdev);
6515 dev_err(rdev->dev, "failed VCE (%d) init.\n", r);
6517 * At this point rdev->vce.vcpu_bo is NULL which trickles down
6518 * to early fails si_vce_start() and thus nothing happens
6519 * there. So it is pointless to try to go through that code
6520 * hence why we disable vce here.
6522 rdev->has_vce = false;
6525 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_obj = NULL;
6526 r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE1_INDEX], 4096);
6527 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_obj = NULL;
6528 r600_ring_init(rdev, &rdev->ring[TN_RING_TYPE_VCE2_INDEX], 4096);
6531 static void si_vce_start(struct radeon_device *rdev)
6538 r = radeon_vce_resume(rdev);
6540 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
6543 r = vce_v1_0_resume(rdev);
6545 dev_err(rdev->dev, "failed VCE resume (%d).\n", r);
6548 r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE1_INDEX);
6550 dev_err(rdev->dev, "failed initializing VCE1 fences (%d).\n", r);
6553 r = radeon_fence_driver_start_ring(rdev, TN_RING_TYPE_VCE2_INDEX);
6555 dev_err(rdev->dev, "failed initializing VCE2 fences (%d).\n", r);
6561 rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
6562 rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
6565 static void si_vce_resume(struct radeon_device *rdev)
6567 struct radeon_ring *ring;
6570 if (!rdev->has_vce || !rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size)
6573 ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
6574 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP);
6576 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
6579 ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
6580 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, VCE_CMD_NO_OP);
6582 dev_err(rdev->dev, "failed initializing VCE1 ring (%d).\n", r);
6585 r = vce_v1_0_init(rdev);
6587 dev_err(rdev->dev, "failed initializing VCE (%d).\n", r);
6592 static int si_startup(struct radeon_device *rdev)
6594 struct radeon_ring *ring;
6597 /* enable pcie gen2/3 link */
6598 si_pcie_gen3_enable(rdev);
6600 si_program_aspm(rdev);
6602 /* scratch needs to be initialized before MC */
6603 r = r600_vram_scratch_init(rdev);
6607 si_mc_program(rdev);
6609 if (!rdev->pm.dpm_enabled) {
6610 r = si_mc_load_microcode(rdev);
6612 DRM_ERROR("Failed to load MC firmware!\n");
6617 r = si_pcie_gart_enable(rdev);
6622 /* allocate rlc buffers */
6623 if (rdev->family == CHIP_VERDE) {
6624 rdev->rlc.reg_list = verde_rlc_save_restore_register_list;
6625 rdev->rlc.reg_list_size =
6626 (u32)ARRAY_SIZE(verde_rlc_save_restore_register_list);
6628 rdev->rlc.cs_data = si_cs_data;
6629 r = sumo_rlc_init(rdev);
6631 DRM_ERROR("Failed to init rlc BOs!\n");
6635 /* allocate wb buffer */
6636 r = radeon_wb_init(rdev);
6640 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
6642 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6646 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6648 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6652 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6654 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6658 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
6660 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6664 r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6666 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6674 if (!rdev->irq.installed) {
6675 r = radeon_irq_kms_init(rdev);
6680 r = si_irq_init(rdev);
6682 DRM_ERROR("radeon: IH init failed (%d).\n", r);
6683 radeon_irq_kms_fini(rdev);
6688 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6689 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
6694 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6695 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
6700 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6701 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
6706 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6707 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
6708 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
6712 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6713 r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
6714 DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
6718 r = si_cp_load_microcode(rdev);
6721 r = si_cp_resume(rdev);
6725 r = cayman_dma_resume(rdev);
6729 si_uvd_resume(rdev);
6730 si_vce_resume(rdev);
6732 r = radeon_ib_pool_init(rdev);
6734 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
6738 r = radeon_vm_manager_init(rdev);
6740 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
6744 r = radeon_audio_init(rdev);
6751 int si_resume(struct radeon_device *rdev)
6755 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
6756 * posting will perform necessary task to bring back GPU into good
6760 atom_asic_init(rdev->mode_info.atom_context);
6762 /* init golden registers */
6763 si_init_golden_registers(rdev);
6765 if (rdev->pm.pm_method == PM_METHOD_DPM)
6766 radeon_pm_resume(rdev);
6768 rdev->accel_working = true;
6769 r = si_startup(rdev);
6771 DRM_ERROR("si startup failed on resume\n");
6772 rdev->accel_working = false;
6780 int si_suspend(struct radeon_device *rdev)
6782 radeon_pm_suspend(rdev);
6783 radeon_audio_fini(rdev);
6784 radeon_vm_manager_fini(rdev);
6785 si_cp_enable(rdev, false);
6786 cayman_dma_stop(rdev);
6787 if (rdev->has_uvd) {
6788 radeon_uvd_suspend(rdev);
6789 uvd_v1_0_fini(rdev);
6792 radeon_vce_suspend(rdev);
6795 si_irq_suspend(rdev);
6796 radeon_wb_disable(rdev);
6797 si_pcie_gart_disable(rdev);
6801 /* Plan is to move initialization in that function and use
6802 * helper function so that radeon_device_init pretty much
6803 * do nothing more than calling asic specific function. This
6804 * should also allow to remove a bunch of callback function
6807 int si_init(struct radeon_device *rdev)
6809 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6813 if (!radeon_get_bios(rdev)) {
6814 if (ASIC_IS_AVIVO(rdev))
6817 /* Must be an ATOMBIOS */
6818 if (!rdev->is_atom_bios) {
6819 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
6822 r = radeon_atombios_init(rdev);
6826 /* Post card if necessary */
6827 if (!radeon_card_posted(rdev)) {
6829 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
6832 DRM_INFO("GPU not posted. posting now...\n");
6833 atom_asic_init(rdev->mode_info.atom_context);
6835 /* init golden registers */
6836 si_init_golden_registers(rdev);
6837 /* Initialize scratch registers */
6838 si_scratch_init(rdev);
6839 /* Initialize surface registers */
6840 radeon_surface_init(rdev);
6841 /* Initialize clocks */
6842 radeon_get_clock_info(rdev_to_drm(rdev));
6845 radeon_fence_driver_init(rdev);
6847 /* initialize memory controller */
6848 r = si_mc_init(rdev);
6851 /* Memory manager */
6852 r = radeon_bo_init(rdev);
6856 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
6857 !rdev->rlc_fw || !rdev->mc_fw) {
6858 r = si_init_microcode(rdev);
6860 DRM_ERROR("Failed to load firmware!\n");
6865 /* Initialize power management */
6866 radeon_pm_init(rdev);
6868 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6869 ring->ring_obj = NULL;
6870 r600_ring_init(rdev, ring, 1024 * 1024);
6872 ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6873 ring->ring_obj = NULL;
6874 r600_ring_init(rdev, ring, 1024 * 1024);
6876 ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
6877 ring->ring_obj = NULL;
6878 r600_ring_init(rdev, ring, 1024 * 1024);
6880 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
6881 ring->ring_obj = NULL;
6882 r600_ring_init(rdev, ring, 64 * 1024);
6884 ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
6885 ring->ring_obj = NULL;
6886 r600_ring_init(rdev, ring, 64 * 1024);
6891 rdev->ih.ring_obj = NULL;
6892 r600_ih_ring_init(rdev, 64 * 1024);
6894 r = r600_pcie_gart_init(rdev);
6898 rdev->accel_working = true;
6899 r = si_startup(rdev);
6901 dev_err(rdev->dev, "disabling GPU acceleration\n");
6903 cayman_dma_fini(rdev);
6905 sumo_rlc_fini(rdev);
6906 radeon_wb_fini(rdev);
6907 radeon_ib_pool_fini(rdev);
6908 radeon_vm_manager_fini(rdev);
6909 radeon_irq_kms_fini(rdev);
6910 si_pcie_gart_fini(rdev);
6911 rdev->accel_working = false;
6914 /* Don't start up if the MC ucode is missing.
6915 * The default clocks and voltages before the MC ucode
6916 * is loaded are not suffient for advanced operations.
6919 DRM_ERROR("radeon: MC ucode required for NI+.\n");
6926 void si_fini(struct radeon_device *rdev)
6928 radeon_pm_fini(rdev);
6930 cayman_dma_fini(rdev);
6934 sumo_rlc_fini(rdev);
6935 radeon_wb_fini(rdev);
6936 radeon_vm_manager_fini(rdev);
6937 radeon_ib_pool_fini(rdev);
6938 radeon_irq_kms_fini(rdev);
6939 if (rdev->has_uvd) {
6940 uvd_v1_0_fini(rdev);
6941 radeon_uvd_fini(rdev);
6944 radeon_vce_fini(rdev);
6945 si_pcie_gart_fini(rdev);
6946 r600_vram_scratch_fini(rdev);
6947 radeon_gem_fini(rdev);
6948 radeon_fence_driver_fini(rdev);
6949 radeon_bo_fini(rdev);
6950 radeon_atombios_fini(rdev);
6956 * si_get_gpu_clock_counter - return GPU clock counter snapshot
6958 * @rdev: radeon_device pointer
6960 * Fetches a GPU clock counter snapshot (SI).
6961 * Returns the 64 bit clock counter snapshot.
6963 uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
6967 mutex_lock(&rdev->gpu_clock_mutex);
6968 WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
6969 clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
6970 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
6971 mutex_unlock(&rdev->gpu_clock_mutex);
6975 int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
6977 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
6980 /* bypass vclk and dclk with bclk */
6981 WREG32_P(CG_UPLL_FUNC_CNTL_2,
6982 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
6983 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
6985 /* put PLL in bypass mode */
6986 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
6988 if (!vclk || !dclk) {
6989 /* keep the Bypass mode */
6993 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
6994 16384, 0x03FFFFFF, 0, 128, 5,
6995 &fb_div, &vclk_div, &dclk_div);
6999 /* set RESET_ANTI_MUX to 0 */
7000 WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
7002 /* set VCO_MODE to 1 */
7003 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
7005 /* disable sleep mode */
7006 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
7008 /* deassert UPLL_RESET */
7009 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
7013 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
7017 /* assert UPLL_RESET again */
7018 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
7020 /* disable spread spectrum. */
7021 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
7023 /* set feedback divider */
7024 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
7026 /* set ref divider to 0 */
7027 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
7029 if (fb_div < 307200)
7030 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
7032 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
7034 /* set PDIV_A and PDIV_B */
7035 WREG32_P(CG_UPLL_FUNC_CNTL_2,
7036 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
7037 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
7039 /* give the PLL some time to settle */
7042 /* deassert PLL_RESET */
7043 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
7047 /* switch from bypass mode to normal mode */
7048 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
7050 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
7054 /* switch VCLK and DCLK selection */
7055 WREG32_P(CG_UPLL_FUNC_CNTL_2,
7056 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
7057 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
7064 static void si_pcie_gen3_enable(struct radeon_device *rdev)
7066 struct pci_dev *root = rdev->pdev->bus->self;
7067 enum pci_bus_speed speed_cap;
7068 u32 speed_cntl, current_data_rate;
7072 if (pci_is_root_bus(rdev->pdev->bus))
7075 if (radeon_pcie_gen2 == 0)
7078 if (rdev->flags & RADEON_IS_IGP)
7081 if (!(rdev->flags & RADEON_IS_PCIE))
7084 speed_cap = pcie_get_speed_cap(root);
7085 if (speed_cap == PCI_SPEED_UNKNOWN)
7088 if ((speed_cap != PCIE_SPEED_8_0GT) &&
7089 (speed_cap != PCIE_SPEED_5_0GT))
7092 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7093 current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
7094 LC_CURRENT_DATA_RATE_SHIFT;
7095 if (speed_cap == PCIE_SPEED_8_0GT) {
7096 if (current_data_rate == 2) {
7097 DRM_INFO("PCIE gen 3 link speeds already enabled\n");
7100 DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
7101 } else if (speed_cap == PCIE_SPEED_5_0GT) {
7102 if (current_data_rate == 1) {
7103 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
7106 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
7109 if (!pci_is_pcie(root) || !pci_is_pcie(rdev->pdev))
7112 if (speed_cap == PCIE_SPEED_8_0GT) {
7113 /* re-try equalization if gen3 is not already enabled */
7114 if (current_data_rate != 2) {
7115 u16 bridge_cfg, gpu_cfg;
7116 u16 bridge_cfg2, gpu_cfg2;
7117 u32 max_lw, current_lw, tmp;
7119 pcie_capability_set_word(root, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_HAWD);
7120 pcie_capability_set_word(rdev->pdev, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_HAWD);
7122 tmp = RREG32_PCIE(PCIE_LC_STATUS1);
7123 max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
7124 current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
7126 if (current_lw < max_lw) {
7127 tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7128 if (tmp & LC_RENEGOTIATION_SUPPORT) {
7129 tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
7130 tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
7131 tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
7132 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
7136 for (i = 0; i < 10; i++) {
7138 pcie_capability_read_word(rdev->pdev,
7141 if (tmp16 & PCI_EXP_DEVSTA_TRPND)
7144 pcie_capability_read_word(root, PCI_EXP_LNKCTL,
7146 pcie_capability_read_word(rdev->pdev,
7150 pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
7152 pcie_capability_read_word(rdev->pdev,
7156 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7157 tmp |= LC_SET_QUIESCE;
7158 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7160 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7162 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7167 pcie_capability_clear_and_set_word(root, PCI_EXP_LNKCTL,
7168 PCI_EXP_LNKCTL_HAWD,
7170 PCI_EXP_LNKCTL_HAWD);
7171 pcie_capability_clear_and_set_word(rdev->pdev, PCI_EXP_LNKCTL,
7172 PCI_EXP_LNKCTL_HAWD,
7174 PCI_EXP_LNKCTL_HAWD);
7177 pcie_capability_clear_and_set_word(root, PCI_EXP_LNKCTL2,
7178 PCI_EXP_LNKCTL2_ENTER_COMP |
7179 PCI_EXP_LNKCTL2_TX_MARGIN,
7181 (PCI_EXP_LNKCTL2_ENTER_COMP |
7182 PCI_EXP_LNKCTL2_TX_MARGIN));
7183 pcie_capability_clear_and_set_word(rdev->pdev, PCI_EXP_LNKCTL2,
7184 PCI_EXP_LNKCTL2_ENTER_COMP |
7185 PCI_EXP_LNKCTL2_TX_MARGIN,
7187 (PCI_EXP_LNKCTL2_ENTER_COMP |
7188 PCI_EXP_LNKCTL2_TX_MARGIN));
7190 tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7191 tmp &= ~LC_SET_QUIESCE;
7192 WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7197 /* set the link speed */
7198 speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
7199 speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
7200 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7203 if (speed_cap == PCIE_SPEED_8_0GT)
7204 tmp16 |= PCI_EXP_LNKCTL2_TLS_8_0GT; /* gen3 */
7205 else if (speed_cap == PCIE_SPEED_5_0GT)
7206 tmp16 |= PCI_EXP_LNKCTL2_TLS_5_0GT; /* gen2 */
7208 tmp16 |= PCI_EXP_LNKCTL2_TLS_2_5GT; /* gen1 */
7209 pcie_capability_clear_and_set_word(rdev->pdev, PCI_EXP_LNKCTL2,
7210 PCI_EXP_LNKCTL2_TLS, tmp16);
7212 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7213 speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
7214 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7216 for (i = 0; i < rdev->usec_timeout; i++) {
7217 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7218 if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
7224 static void si_program_aspm(struct radeon_device *rdev)
7227 bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
7228 bool disable_clkreq = false;
7230 if (radeon_aspm == 0)
7233 if (!(rdev->flags & RADEON_IS_PCIE))
7236 orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7237 data &= ~LC_XMIT_N_FTS_MASK;
7238 data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
7240 WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
7242 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
7243 data |= LC_GO_TO_RECOVERY;
7245 WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
7247 orig = data = RREG32_PCIE(PCIE_P_CNTL);
7248 data |= P_IGNORE_EDB_ERR;
7250 WREG32_PCIE(PCIE_P_CNTL, data);
7252 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7253 data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
7254 data |= LC_PMI_TO_L1_DIS;
7256 data |= LC_L0S_INACTIVITY(7);
7259 data |= LC_L1_INACTIVITY(7);
7260 data &= ~LC_PMI_TO_L1_DIS;
7262 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7264 if (!disable_plloff_in_l1) {
7265 bool clk_req_support;
7267 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7268 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7269 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7271 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7273 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7274 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7275 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7277 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7279 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7280 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7281 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7283 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7285 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7286 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7287 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7289 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7291 if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) {
7292 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7293 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7295 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7297 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7298 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7300 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7302 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2);
7303 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7305 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data);
7307 orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3);
7308 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7310 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data);
7312 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7313 data &= ~PLL_RAMP_UP_TIME_0_MASK;
7315 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7317 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7318 data &= ~PLL_RAMP_UP_TIME_1_MASK;
7320 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7322 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2);
7323 data &= ~PLL_RAMP_UP_TIME_2_MASK;
7325 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data);
7327 orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3);
7328 data &= ~PLL_RAMP_UP_TIME_3_MASK;
7330 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data);
7332 orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7333 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
7334 data |= LC_DYN_LANES_PWR_STATE(3);
7336 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
7338 orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL);
7339 data &= ~LS2_EXIT_TIME_MASK;
7340 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7341 data |= LS2_EXIT_TIME(5);
7343 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
7345 orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL);
7346 data &= ~LS2_EXIT_TIME_MASK;
7347 if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7348 data |= LS2_EXIT_TIME(5);
7350 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
7352 if (!disable_clkreq &&
7353 !pci_is_root_bus(rdev->pdev->bus)) {
7354 struct pci_dev *root = rdev->pdev->bus->self;
7357 clk_req_support = false;
7358 pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
7359 if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
7360 clk_req_support = true;
7362 clk_req_support = false;
7365 if (clk_req_support) {
7366 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
7367 data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
7369 WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
7371 orig = data = RREG32(THM_CLK_CNTL);
7372 data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
7373 data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
7375 WREG32(THM_CLK_CNTL, data);
7377 orig = data = RREG32(MISC_CLK_CNTL);
7378 data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
7379 data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
7381 WREG32(MISC_CLK_CNTL, data);
7383 orig = data = RREG32(CG_CLKPIN_CNTL);
7384 data &= ~BCLK_AS_XCLK;
7386 WREG32(CG_CLKPIN_CNTL, data);
7388 orig = data = RREG32(CG_CLKPIN_CNTL_2);
7389 data &= ~FORCE_BIF_REFCLK_EN;
7391 WREG32(CG_CLKPIN_CNTL_2, data);
7393 orig = data = RREG32(MPLL_BYPASSCLK_SEL);
7394 data &= ~MPLL_CLKOUT_SEL_MASK;
7395 data |= MPLL_CLKOUT_SEL(4);
7397 WREG32(MPLL_BYPASSCLK_SEL, data);
7399 orig = data = RREG32(SPLL_CNTL_MODE);
7400 data &= ~SPLL_REFCLK_SEL_MASK;
7402 WREG32(SPLL_CNTL_MODE, data);
7407 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7410 orig = data = RREG32_PCIE(PCIE_CNTL2);
7411 data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
7413 WREG32_PCIE(PCIE_CNTL2, data);
7416 data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7417 if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
7418 data = RREG32_PCIE(PCIE_LC_STATUS1);
7419 if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
7420 orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7421 data &= ~LC_L0S_INACTIVITY_MASK;
7423 WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7429 static int si_vce_send_vcepll_ctlreq(struct radeon_device *rdev)
7433 /* make sure VCEPLL_CTLREQ is deasserted */
7434 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
7438 /* assert UPLL_CTLREQ */
7439 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, UPLL_CTLREQ_MASK, ~UPLL_CTLREQ_MASK);
7441 /* wait for CTLACK and CTLACK2 to get asserted */
7442 for (i = 0; i < 100; ++i) {
7443 uint32_t mask = UPLL_CTLACK_MASK | UPLL_CTLACK2_MASK;
7444 if ((RREG32_SMC(CG_VCEPLL_FUNC_CNTL) & mask) == mask)
7449 /* deassert UPLL_CTLREQ */
7450 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
7453 DRM_ERROR("Timeout setting UVD clocks!\n");
7460 int si_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
7462 unsigned fb_div = 0, evclk_div = 0, ecclk_div = 0;
7465 /* bypass evclk and ecclk with bclk */
7466 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
7467 EVCLK_SRC_SEL(1) | ECCLK_SRC_SEL(1),
7468 ~(EVCLK_SRC_SEL_MASK | ECCLK_SRC_SEL_MASK));
7470 /* put PLL in bypass mode */
7471 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_BYPASS_EN_MASK,
7472 ~VCEPLL_BYPASS_EN_MASK);
7474 if (!evclk || !ecclk) {
7475 /* keep the Bypass mode, put PLL to sleep */
7476 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK,
7477 ~VCEPLL_SLEEP_MASK);
7481 r = radeon_uvd_calc_upll_dividers(rdev, evclk, ecclk, 125000, 250000,
7482 16384, 0x03FFFFFF, 0, 128, 5,
7483 &fb_div, &evclk_div, &ecclk_div);
7487 /* set RESET_ANTI_MUX to 0 */
7488 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
7490 /* set VCO_MODE to 1 */
7491 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_VCO_MODE_MASK,
7492 ~VCEPLL_VCO_MODE_MASK);
7494 /* toggle VCEPLL_SLEEP to 1 then back to 0 */
7495 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK,
7496 ~VCEPLL_SLEEP_MASK);
7497 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_SLEEP_MASK);
7499 /* deassert VCEPLL_RESET */
7500 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK);
7504 r = si_vce_send_vcepll_ctlreq(rdev);
7508 /* assert VCEPLL_RESET again */
7509 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_RESET_MASK, ~VCEPLL_RESET_MASK);
7511 /* disable spread spectrum. */
7512 WREG32_SMC_P(CG_VCEPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
7514 /* set feedback divider */
7515 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_3, VCEPLL_FB_DIV(fb_div), ~VCEPLL_FB_DIV_MASK);
7517 /* set ref divider to 0 */
7518 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_REF_DIV_MASK);
7520 /* set PDIV_A and PDIV_B */
7521 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
7522 VCEPLL_PDIV_A(evclk_div) | VCEPLL_PDIV_B(ecclk_div),
7523 ~(VCEPLL_PDIV_A_MASK | VCEPLL_PDIV_B_MASK));
7525 /* give the PLL some time to settle */
7528 /* deassert PLL_RESET */
7529 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK);
7533 /* switch from bypass mode to normal mode */
7534 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_BYPASS_EN_MASK);
7536 r = si_vce_send_vcepll_ctlreq(rdev);
7540 /* switch VCLK and DCLK selection */
7541 WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
7542 EVCLK_SRC_SEL(16) | ECCLK_SRC_SEL(16),
7543 ~(EVCLK_SRC_SEL_MASK | ECCLK_SRC_SEL_MASK));
projects / linux.git / blob