drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23 #include "amdgpu.h"
  24 #include "mmhub_v1_0.h"
  25
  26 #include "vega10/soc15ip.h"
  27 #include "vega10/MMHUB/mmhub_1_0_offset.h"
  28 #include "vega10/MMHUB/mmhub_1_0_sh_mask.h"
  29 #include "vega10/MMHUB/mmhub_1_0_default.h"
  30 #include "vega10/ATHUB/athub_1_0_offset.h"
  31 #include "vega10/ATHUB/athub_1_0_sh_mask.h"
  32 #include "vega10/ATHUB/athub_1_0_default.h"
  33 #include "vega10/vega10_enum.h"
  34
  35 #include "soc15_common.h"
  36
  37 #define mmDAGB0_CNTL_MISC2_RV 0x008f
  38 #define mmDAGB0_CNTL_MISC2_RV_BASE_IDX 0
  39
  40 u64 mmhub_v1_0_get_fb_location(struct amdgpu_device *adev)
  41 {
  42         u64 base = RREG32_SOC15(MMHUB, 0, mmMC_VM_FB_LOCATION_BASE);
  43
  44         base &= MC_VM_FB_LOCATION_BASE__FB_BASE_MASK;
  45         base <<= 24;
  46
  47         return base;
  48 }
  49
  50 static void mmhub_v1_0_init_gart_pt_regs(struct amdgpu_device *adev)
  51 {
  52         uint64_t value;
  53
  54         BUG_ON(adev->gart.table_addr & (~0x0000FFFFFFFFF000ULL));
  55         value = adev->gart.table_addr - adev->mc.vram_start +
  56                 adev->vm_manager.vram_base_offset;
  57         value &= 0x0000FFFFFFFFF000ULL;
  58         value |= 0x1; /* valid bit */
  59
  60         WREG32_SOC15(MMHUB, 0, mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32,
  61                      lower_32_bits(value));
  62
  63         WREG32_SOC15(MMHUB, 0, mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32,
  64                      upper_32_bits(value));
  65 }
  66
  67 static void mmhub_v1_0_init_gart_aperture_regs(struct amdgpu_device *adev)
  68 {
  69         mmhub_v1_0_init_gart_pt_regs(adev);
  70
  71         WREG32_SOC15(MMHUB, 0, mmVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32,
  72                      (u32)(adev->mc.gtt_start >> 12));
  73         WREG32_SOC15(MMHUB, 0, mmVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32,
  74                      (u32)(adev->mc.gtt_start >> 44));
  75
  76         WREG32_SOC15(MMHUB, 0, mmVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32,
  77                      (u32)(adev->mc.gtt_end >> 12));
  78         WREG32_SOC15(MMHUB, 0, mmVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32,
  79                      (u32)(adev->mc.gtt_end >> 44));
  80 }
  81
  82 static void mmhub_v1_0_init_system_aperture_regs(struct amdgpu_device *adev)
  83 {
  84         uint64_t value;
  85         uint32_t tmp;
  86
  87         /* Disable AGP. */
  88         WREG32_SOC15(MMHUB, 0, mmMC_VM_AGP_BASE, 0);
  89         WREG32_SOC15(MMHUB, 0, mmMC_VM_AGP_TOP, 0);
  90         WREG32_SOC15(MMHUB, 0, mmMC_VM_AGP_BOT, 0x00FFFFFF);
  91
  92         /* Program the system aperture low logical page number. */
  93         WREG32_SOC15(MMHUB, 0, mmMC_VM_SYSTEM_APERTURE_LOW_ADDR,
  94                      adev->mc.vram_start >> 18);
  95         WREG32_SOC15(MMHUB, 0, mmMC_VM_SYSTEM_APERTURE_HIGH_ADDR,
  96                      adev->mc.vram_end >> 18);
  97
  98         /* Set default page address. */
  99         value = adev->vram_scratch.gpu_addr - adev->mc.vram_start +
 100                 adev->vm_manager.vram_base_offset;
 101         WREG32_SOC15(MMHUB, 0, mmMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB,
 102                      (u32)(value >> 12));
 103         WREG32_SOC15(MMHUB, 0, mmMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB,
 104                      (u32)(value >> 44));
 105
 106         /* Program "protection fault". */
 107         WREG32_SOC15(MMHUB, 0, mmVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32,
 108                      (u32)(adev->dummy_page.addr >> 12));
 109         WREG32_SOC15(MMHUB, 0, mmVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32,
 110                      (u32)((u64)adev->dummy_page.addr >> 44));
 111
 112         tmp = RREG32_SOC15(MMHUB, 0, mmVM_L2_PROTECTION_FAULT_CNTL2);
 113         tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL2,
 114                             ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1);
 115         WREG32_SOC15(MMHUB, 0, mmVM_L2_PROTECTION_FAULT_CNTL2, tmp);
 116 }
 117
 118 static void mmhub_v1_0_init_tlb_regs(struct amdgpu_device *adev)
 119 {
 120         uint32_t tmp;
 121
 122         /* Setup TLB control */
 123         tmp = RREG32_SOC15(MMHUB, 0, mmMC_VM_MX_L1_TLB_CNTL);
 124
 125         tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 1);
 126         tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE, 3);
 127         tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL,
 128                             ENABLE_ADVANCED_DRIVER_MODEL, 1);
 129         tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL,
 130                             SYSTEM_APERTURE_UNMAPPED_ACCESS, 0);
 131         tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ECO_BITS, 0);
 132         tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL,
 133                             MTYPE, MTYPE_UC);/* XXX for emulation. */
 134         tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ATC_EN, 1);
 135
 136         WREG32_SOC15(MMHUB, 0, mmMC_VM_MX_L1_TLB_CNTL, tmp);
 137 }
 138
 139 static void mmhub_v1_0_init_cache_regs(struct amdgpu_device *adev)
 140 {
 141         uint32_t tmp;
 142
 143         /* Setup L2 cache */
 144         tmp = RREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL);
 145         tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 1);
 146         tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING, 0);
 147         /* XXX for emulation, Refer to closed source code.*/
 148         tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, L2_PDE0_CACHE_TAG_GENERATION_MODE,
 149                             0);
 150         tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, PDE_FAULT_CLASSIFICATION, 1);
 151         tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, CONTEXT1_IDENTITY_ACCESS_MODE, 1);
 152         tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, IDENTITY_MODE_FRAGMENT_SIZE, 0);
 153         WREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL, tmp);
 154
 155         tmp = RREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL2);
 156         tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 1);
 157         tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_L2_CACHE, 1);
 158         WREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL2, tmp);
 159
 160         tmp = mmVM_L2_CNTL3_DEFAULT;
 161         WREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL3, tmp);
 162
 163         tmp = mmVM_L2_CNTL4_DEFAULT;
 164         tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_PDE_REQUEST_PHYSICAL, 0);
 165         tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_PTE_REQUEST_PHYSICAL, 0);
 166         WREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL4, tmp);
 167 }
 168
 169 static void mmhub_v1_0_enable_system_domain(struct amdgpu_device *adev)
 170 {
 171         uint32_t tmp;
 172
 173         tmp = RREG32_SOC15(MMHUB, 0, mmVM_CONTEXT0_CNTL);
 174         tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1);
 175         tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH, 0);
 176         WREG32_SOC15(MMHUB, 0, mmVM_CONTEXT0_CNTL, tmp);
 177 }
 178
 179 static void mmhub_v1_0_disable_identity_aperture(struct amdgpu_device *adev)
 180 {
 181         WREG32_SOC15(MMHUB, 0, mmVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32,
 182                      0XFFFFFFFF);
 183         WREG32_SOC15(MMHUB, 0, mmVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32,
 184                      0x0000000F);
 185
 186         WREG32_SOC15(MMHUB, 0,
 187                      mmVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32, 0);
 188         WREG32_SOC15(MMHUB, 0,
 189                      mmVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32, 0);
 190
 191         WREG32_SOC15(MMHUB, 0, mmVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32,
 192                      0);
 193         WREG32_SOC15(MMHUB, 0, mmVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32,
 194                      0);
 195 }
 196
 197 static void mmhub_v1_0_setup_vmid_config(struct amdgpu_device *adev)
 198 {
 199         int i;
 200         uint32_t tmp;
 201
 202         for (i = 0; i <= 14; i++) {
 203                 tmp = RREG32_SOC15_OFFSET(MMHUB, 0, mmVM_CONTEXT1_CNTL, i);
 204                 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 205                                 ENABLE_CONTEXT, 1);
 206                 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 207                                 PAGE_TABLE_DEPTH, adev->vm_manager.num_level);
 208                 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 209                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 210                 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 211                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 212                 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 213                                 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 214                 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 215                                 VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 216                 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 217                                 READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 218                 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 219                                 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 220                 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 221                                 EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 222                 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 223                                 PAGE_TABLE_BLOCK_SIZE,
 224                                 adev->vm_manager.block_size - 9);
 225                 WREG32_SOC15_OFFSET(MMHUB, 0, mmVM_CONTEXT1_CNTL, i, tmp);
 226                 WREG32_SOC15_OFFSET(MMHUB, 0, mmVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32, i*2, 0);
 227                 WREG32_SOC15_OFFSET(MMHUB, 0, mmVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32, i*2, 0);
 228                 WREG32_SOC15_OFFSET(MMHUB, 0, mmVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32, i*2,
 229                         lower_32_bits(adev->vm_manager.max_pfn - 1));
 230                 WREG32_SOC15_OFFSET(MMHUB, 0, mmVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32, i*2,
 231                         upper_32_bits(adev->vm_manager.max_pfn - 1));
 232         }
 233 }
 234
 235 static void mmhub_v1_0_program_invalidation(struct amdgpu_device *adev)
 236 {
 237         unsigned i;
 238
 239         for (i = 0; i < 18; ++i) {
 240                 WREG32_SOC15_OFFSET(MMHUB, 0, mmVM_INVALIDATE_ENG0_ADDR_RANGE_LO32,
 241                                     2 * i, 0xffffffff);
 242                 WREG32_SOC15_OFFSET(MMHUB, 0, mmVM_INVALIDATE_ENG0_ADDR_RANGE_HI32,
 243                                     2 * i, 0x1f);
 244         }
 245 }
 246
 247 struct pctl_data {
 248     uint32_t index;
 249     uint32_t data;
 250 };
 251
 252 const struct pctl_data pctl0_data[] = {
 253     {0x0, 0x7a640},
 254     {0x9, 0x2a64a},
 255     {0xd, 0x2a680},
 256     {0x11, 0x6a684},
 257     {0x19, 0xea68e},
 258     {0x29, 0xa69e},
 259     {0x2b, 0x34a6c0},
 260     {0x61, 0x83a707},
 261     {0xe6, 0x8a7a4},
 262     {0xf0, 0x1a7b8},
 263     {0xf3, 0xfa7cc},
 264     {0x104, 0x17a7dd},
 265     {0x11d, 0xa7dc},
 266     {0x11f, 0x12a7f5},
 267     {0x133, 0xa808},
 268     {0x135, 0x12a810},
 269     {0x149, 0x7a82c}
 270 };
 271 #define PCTL0_DATA_LEN (sizeof(pctl0_data)/sizeof(pctl0_data[0]))
 272
 273 #define PCTL0_RENG_EXEC_END_PTR 0x151
 274 #define PCTL0_STCTRL_REG_SAVE_RANGE0_BASE  0xa640
 275 #define PCTL0_STCTRL_REG_SAVE_RANGE0_LIMIT 0xa833
 276
 277 const struct pctl_data pctl1_data[] = {
 278     {0x0, 0x39a000},
 279     {0x3b, 0x44a040},
 280     {0x81, 0x2a08d},
 281     {0x85, 0x6ba094},
 282     {0xf2, 0x18a100},
 283     {0x10c, 0x4a132},
 284     {0x112, 0xca141},
 285     {0x120, 0x2fa158},
 286     {0x151, 0x17a1d0},
 287     {0x16a, 0x1a1e9},
 288     {0x16d, 0x13a1ec},
 289     {0x182, 0x7a201},
 290     {0x18b, 0x3a20a},
 291     {0x190, 0x7a580},
 292     {0x199, 0xa590},
 293     {0x19b, 0x4a594},
 294     {0x1a1, 0x1a59c},
 295     {0x1a4, 0x7a82c},
 296     {0x1ad, 0xfa7cc},
 297     {0x1be, 0x17a7dd},
 298     {0x1d7, 0x12a810}
 299 };
 300 #define PCTL1_DATA_LEN (sizeof(pctl1_data)/sizeof(pctl1_data[0]))
 301
 302 #define PCTL1_RENG_EXEC_END_PTR 0x1ea
 303 #define PCTL1_STCTRL_REG_SAVE_RANGE0_BASE  0xa000
 304 #define PCTL1_STCTRL_REG_SAVE_RANGE0_LIMIT 0xa20d
 305 #define PCTL1_STCTRL_REG_SAVE_RANGE1_BASE  0xa580
 306 #define PCTL1_STCTRL_REG_SAVE_RANGE1_LIMIT 0xa59d
 307 #define PCTL1_STCTRL_REG_SAVE_RANGE2_BASE  0xa82c
 308 #define PCTL1_STCTRL_REG_SAVE_RANGE2_LIMIT 0xa833
 309
 310 static void mmhub_v1_0_power_gating_write_save_ranges(struct amdgpu_device *adev)
 311 {
 312         uint32_t tmp = 0;
 313
 314         /* PCTL0_STCTRL_REGISTER_SAVE_RANGE0 */
 315         tmp = REG_SET_FIELD(tmp, PCTL0_STCTRL_REGISTER_SAVE_RANGE0,
 316                         STCTRL_REGISTER_SAVE_BASE,
 317                         PCTL0_STCTRL_REG_SAVE_RANGE0_BASE);
 318         tmp = REG_SET_FIELD(tmp, PCTL0_STCTRL_REGISTER_SAVE_RANGE0,
 319                         STCTRL_REGISTER_SAVE_LIMIT,
 320                         PCTL0_STCTRL_REG_SAVE_RANGE0_LIMIT);
 321         WREG32_SOC15(MMHUB, 0, mmPCTL0_STCTRL_REGISTER_SAVE_RANGE0, tmp);
 322
 323         /* PCTL1_STCTRL_REGISTER_SAVE_RANGE0 */
 324         tmp = 0;
 325         tmp = REG_SET_FIELD(tmp, PCTL1_STCTRL_REGISTER_SAVE_RANGE0,
 326                         STCTRL_REGISTER_SAVE_BASE,
 327                         PCTL1_STCTRL_REG_SAVE_RANGE0_BASE);
 328         tmp = REG_SET_FIELD(tmp, PCTL1_STCTRL_REGISTER_SAVE_RANGE0,
 329                         STCTRL_REGISTER_SAVE_LIMIT,
 330                         PCTL1_STCTRL_REG_SAVE_RANGE0_LIMIT);
 331         WREG32_SOC15(MMHUB, 0, mmPCTL1_STCTRL_REGISTER_SAVE_RANGE0, tmp);
 332
 333         /* PCTL1_STCTRL_REGISTER_SAVE_RANGE1 */
 334         tmp = 0;
 335         tmp = REG_SET_FIELD(tmp, PCTL1_STCTRL_REGISTER_SAVE_RANGE1,
 336                         STCTRL_REGISTER_SAVE_BASE,
 337                         PCTL1_STCTRL_REG_SAVE_RANGE1_BASE);
 338         tmp = REG_SET_FIELD(tmp, PCTL1_STCTRL_REGISTER_SAVE_RANGE1,
 339                         STCTRL_REGISTER_SAVE_LIMIT,
 340                         PCTL1_STCTRL_REG_SAVE_RANGE1_LIMIT);
 341         WREG32_SOC15(MMHUB, 0, mmPCTL1_STCTRL_REGISTER_SAVE_RANGE1, tmp);
 342
 343         /* PCTL1_STCTRL_REGISTER_SAVE_RANGE2 */
 344         tmp = 0;
 345         tmp = REG_SET_FIELD(tmp, PCTL1_STCTRL_REGISTER_SAVE_RANGE2,
 346                         STCTRL_REGISTER_SAVE_BASE,
 347                         PCTL1_STCTRL_REG_SAVE_RANGE2_BASE);
 348         tmp = REG_SET_FIELD(tmp, PCTL1_STCTRL_REGISTER_SAVE_RANGE2,
 349                         STCTRL_REGISTER_SAVE_LIMIT,
 350                         PCTL1_STCTRL_REG_SAVE_RANGE2_LIMIT);
 351         WREG32_SOC15(MMHUB, 0, mmPCTL1_STCTRL_REGISTER_SAVE_RANGE2, tmp);
 352 }
 353
 354 void mmhub_v1_0_initialize_power_gating(struct amdgpu_device *adev)
 355 {
 356         uint32_t pctl0_misc = 0;
 357         uint32_t pctl0_reng_execute = 0;
 358         uint32_t pctl1_misc = 0;
 359         uint32_t pctl1_reng_execute = 0;
 360         int i = 0;
 361
 362         if (amdgpu_sriov_vf(adev))
 363                 return;
 364
 365         pctl0_misc = RREG32_SOC15(MMHUB, 0, mmPCTL0_MISC);
 366         pctl0_reng_execute = RREG32_SOC15(MMHUB, 0, mmPCTL0_RENG_EXECUTE);
 367         pctl1_misc = RREG32_SOC15(MMHUB, 0, mmPCTL1_MISC);
 368         pctl1_reng_execute = RREG32_SOC15(MMHUB, 0, mmPCTL1_RENG_EXECUTE);
 369
 370         /* Light sleep must be disabled before writing to pctl0 registers */
 371         pctl0_misc &= ~PCTL0_MISC__RENG_MEM_LS_ENABLE_MASK;
 372         WREG32_SOC15(MMHUB, 0, mmPCTL0_MISC, pctl0_misc);
 373
 374         /* Write data used to access ram of register engine */
 375         for (i = 0; i < PCTL0_DATA_LEN; i++) {
 376                 WREG32_SOC15(MMHUB, 0, mmPCTL0_RENG_RAM_INDEX,
 377                         pctl0_data[i].index);
 378                 WREG32_SOC15(MMHUB, 0, mmPCTL0_RENG_RAM_DATA,
 379                         pctl0_data[i].data);
 380         }
 381
 382         /* Set the reng execute end ptr for pctl0 */
 383         pctl0_reng_execute = REG_SET_FIELD(pctl0_reng_execute,
 384                                         PCTL0_RENG_EXECUTE,
 385                                         RENG_EXECUTE_END_PTR,
 386                                         PCTL0_RENG_EXEC_END_PTR);
 387         WREG32_SOC15(MMHUB, 0, mmPCTL0_RENG_EXECUTE, pctl0_reng_execute);
 388
 389         /* Light sleep must be disabled before writing to pctl1 registers */
 390         pctl1_misc &= ~PCTL1_MISC__RENG_MEM_LS_ENABLE_MASK;
 391         WREG32_SOC15(MMHUB, 0, mmPCTL1_MISC, pctl1_misc);
 392
 393         /* Write data used to access ram of register engine */
 394         for (i = 0; i < PCTL1_DATA_LEN; i++) {
 395                 WREG32_SOC15(MMHUB, 0, mmPCTL1_RENG_RAM_INDEX,
 396                         pctl1_data[i].index);
 397                 WREG32_SOC15(MMHUB, 0, mmPCTL1_RENG_RAM_DATA,
 398                         pctl1_data[i].data);
 399         }
 400
 401         /* Set the reng execute end ptr for pctl1 */
 402         pctl1_reng_execute = REG_SET_FIELD(pctl1_reng_execute,
 403                                         PCTL1_RENG_EXECUTE,
 404                                         RENG_EXECUTE_END_PTR,
 405                                         PCTL1_RENG_EXEC_END_PTR);
 406         WREG32_SOC15(MMHUB, 0, mmPCTL1_RENG_EXECUTE, pctl1_reng_execute);
 407
 408         mmhub_v1_0_power_gating_write_save_ranges(adev);
 409
 410         /* Re-enable light sleep */
 411         pctl0_misc |= PCTL0_MISC__RENG_MEM_LS_ENABLE_MASK;
 412         WREG32_SOC15(MMHUB, 0, mmPCTL0_MISC, pctl0_misc);
 413         pctl1_misc |= PCTL1_MISC__RENG_MEM_LS_ENABLE_MASK;
 414         WREG32_SOC15(MMHUB, 0, mmPCTL1_MISC, pctl1_misc);
 415 }
 416
 417 void mmhub_v1_0_update_power_gating(struct amdgpu_device *adev,
 418                                 bool enable)
 419 {
 420         uint32_t pctl0_reng_execute = 0;
 421         uint32_t pctl1_reng_execute = 0;
 422
 423         if (amdgpu_sriov_vf(adev))
 424                 return;
 425
 426         pctl0_reng_execute = RREG32_SOC15(MMHUB, 0, mmPCTL0_RENG_EXECUTE);
 427         pctl1_reng_execute = RREG32_SOC15(MMHUB, 0, mmPCTL1_RENG_EXECUTE);
 428
 429         if (enable && adev->pg_flags & AMD_PG_SUPPORT_MMHUB) {
 430                 pctl0_reng_execute = REG_SET_FIELD(pctl0_reng_execute,
 431                                                 PCTL0_RENG_EXECUTE,
 432                                                 RENG_EXECUTE_ON_PWR_UP, 1);
 433                 pctl0_reng_execute = REG_SET_FIELD(pctl0_reng_execute,
 434                                                 PCTL0_RENG_EXECUTE,
 435                                                 RENG_EXECUTE_ON_REG_UPDATE, 1);
 436                 WREG32_SOC15(MMHUB, 0, mmPCTL0_RENG_EXECUTE, pctl0_reng_execute);
 437
 438                 pctl1_reng_execute = REG_SET_FIELD(pctl1_reng_execute,
 439                                                 PCTL1_RENG_EXECUTE,
 440                                                 RENG_EXECUTE_ON_PWR_UP, 1);
 441                 pctl1_reng_execute = REG_SET_FIELD(pctl1_reng_execute,
 442                                                 PCTL1_RENG_EXECUTE,
 443                                                 RENG_EXECUTE_ON_REG_UPDATE, 1);
 444                 WREG32_SOC15(MMHUB, 0, mmPCTL1_RENG_EXECUTE, pctl1_reng_execute);
 445
 446         } else {
 447                 pctl0_reng_execute = REG_SET_FIELD(pctl0_reng_execute,
 448                                                 PCTL0_RENG_EXECUTE,
 449                                                 RENG_EXECUTE_ON_PWR_UP, 0);
 450                 pctl0_reng_execute = REG_SET_FIELD(pctl0_reng_execute,
 451                                                 PCTL0_RENG_EXECUTE,
 452                                                 RENG_EXECUTE_ON_REG_UPDATE, 0);
 453                 WREG32_SOC15(MMHUB, 0, mmPCTL0_RENG_EXECUTE, pctl0_reng_execute);
 454
 455                 pctl1_reng_execute = REG_SET_FIELD(pctl1_reng_execute,
 456                                                 PCTL1_RENG_EXECUTE,
 457                                                 RENG_EXECUTE_ON_PWR_UP, 0);
 458                 pctl1_reng_execute = REG_SET_FIELD(pctl1_reng_execute,
 459                                                 PCTL1_RENG_EXECUTE,
 460                                                 RENG_EXECUTE_ON_REG_UPDATE, 0);
 461                 WREG32_SOC15(MMHUB, 0, mmPCTL1_RENG_EXECUTE, pctl1_reng_execute);
 462         }
 463 }
 464
 465 int mmhub_v1_0_gart_enable(struct amdgpu_device *adev)
 466 {
 467         if (amdgpu_sriov_vf(adev)) {
 468                 /*
 469                  * MC_VM_FB_LOCATION_BASE/TOP is NULL for VF, becuase they are
 470                  * VF copy registers so vbios post doesn't program them, for
 471                  * SRIOV driver need to program them
 472                  */
 473                 WREG32_SOC15(MMHUB, 0, mmMC_VM_FB_LOCATION_BASE,
 474                              adev->mc.vram_start >> 24);
 475                 WREG32_SOC15(MMHUB, 0, mmMC_VM_FB_LOCATION_TOP,
 476                              adev->mc.vram_end >> 24);
 477         }
 478
 479         /* GART Enable. */
 480         mmhub_v1_0_init_gart_aperture_regs(adev);
 481         mmhub_v1_0_init_system_aperture_regs(adev);
 482         mmhub_v1_0_init_tlb_regs(adev);
 483         mmhub_v1_0_init_cache_regs(adev);
 484
 485         mmhub_v1_0_enable_system_domain(adev);
 486         mmhub_v1_0_disable_identity_aperture(adev);
 487         mmhub_v1_0_setup_vmid_config(adev);
 488         mmhub_v1_0_program_invalidation(adev);
 489
 490         return 0;
 491 }
 492
 493 void mmhub_v1_0_gart_disable(struct amdgpu_device *adev)
 494 {
 495         u32 tmp;
 496         u32 i;
 497
 498         /* Disable all tables */
 499         for (i = 0; i < 16; i++)
 500                 WREG32_SOC15_OFFSET(MMHUB, 0, mmVM_CONTEXT0_CNTL, i, 0);
 501
 502         /* Setup TLB control */
 503         tmp = RREG32_SOC15(MMHUB, 0, mmMC_VM_MX_L1_TLB_CNTL);
 504         tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 0);
 505         tmp = REG_SET_FIELD(tmp,
 506                                 MC_VM_MX_L1_TLB_CNTL,
 507                                 ENABLE_ADVANCED_DRIVER_MODEL,
 508                                 0);
 509         WREG32_SOC15(MMHUB, 0, mmMC_VM_MX_L1_TLB_CNTL, tmp);
 510
 511         /* Setup L2 cache */
 512         tmp = RREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL);
 513         tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 0);
 514         WREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL, tmp);
 515         WREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL3, 0);
 516 }
 517
 518 /**
 519  * mmhub_v1_0_set_fault_enable_default - update GART/VM fault handling
 520  *
 521  * @adev: amdgpu_device pointer
 522  * @value: true redirects VM faults to the default page
 523  */
 524 void mmhub_v1_0_set_fault_enable_default(struct amdgpu_device *adev, bool value)
 525 {
 526         u32 tmp;
 527         tmp = RREG32_SOC15(MMHUB, 0, mmVM_L2_PROTECTION_FAULT_CNTL);
 528         tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
 529                         RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
 530         tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
 531                         PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value);
 532         tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
 533                         PDE1_PROTECTION_FAULT_ENABLE_DEFAULT, value);
 534         tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
 535                         PDE2_PROTECTION_FAULT_ENABLE_DEFAULT, value);
 536         tmp = REG_SET_FIELD(tmp,
 537                         VM_L2_PROTECTION_FAULT_CNTL,
 538                         TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT,
 539                         value);
 540         tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
 541                         NACK_PROTECTION_FAULT_ENABLE_DEFAULT, value);
 542         tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
 543                         DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
 544         tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
 545                         VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value);
 546         tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
 547                         READ_PROTECTION_FAULT_ENABLE_DEFAULT, value);
 548         tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
 549                         WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
 550         tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
 551                         EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
 552         WREG32_SOC15(MMHUB, 0, mmVM_L2_PROTECTION_FAULT_CNTL, tmp);
 553 }
 554
 555 void mmhub_v1_0_init(struct amdgpu_device *adev)
 556 {
 557         struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB];
 558
 559         hub->ctx0_ptb_addr_lo32 =
 560                 SOC15_REG_OFFSET(MMHUB, 0,
 561                                  mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32);
 562         hub->ctx0_ptb_addr_hi32 =
 563                 SOC15_REG_OFFSET(MMHUB, 0,
 564                                  mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32);
 565         hub->vm_inv_eng0_req =
 566                 SOC15_REG_OFFSET(MMHUB, 0, mmVM_INVALIDATE_ENG0_REQ);
 567         hub->vm_inv_eng0_ack =
 568                 SOC15_REG_OFFSET(MMHUB, 0, mmVM_INVALIDATE_ENG0_ACK);
 569         hub->vm_context0_cntl =
 570                 SOC15_REG_OFFSET(MMHUB, 0, mmVM_CONTEXT0_CNTL);
 571         hub->vm_l2_pro_fault_status =
 572                 SOC15_REG_OFFSET(MMHUB, 0, mmVM_L2_PROTECTION_FAULT_STATUS);
 573         hub->vm_l2_pro_fault_cntl =
 574                 SOC15_REG_OFFSET(MMHUB, 0, mmVM_L2_PROTECTION_FAULT_CNTL);
 575
 576 }
 577
 578 static void mmhub_v1_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
 579                                                         bool enable)
 580 {
 581         uint32_t def, data, def1, data1, def2 = 0, data2 = 0;
 582
 583         def  = data  = RREG32_SOC15(MMHUB, 0, mmATC_L2_MISC_CG);
 584
 585         if (adev->asic_type != CHIP_RAVEN) {
 586                 def1 = data1 = RREG32_SOC15(MMHUB, 0, mmDAGB0_CNTL_MISC2);
 587                 def2 = data2 = RREG32_SOC15(MMHUB, 0, mmDAGB1_CNTL_MISC2);
 588         } else
 589                 def1 = data1 = RREG32_SOC15(MMHUB, 0, mmDAGB0_CNTL_MISC2_RV);
 590
 591         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_MGCG)) {
 592                 data |= ATC_L2_MISC_CG__ENABLE_MASK;
 593
 594                 data1 &= ~(DAGB0_CNTL_MISC2__DISABLE_WRREQ_CG_MASK |
 595                            DAGB0_CNTL_MISC2__DISABLE_WRRET_CG_MASK |
 596                            DAGB0_CNTL_MISC2__DISABLE_RDREQ_CG_MASK |
 597                            DAGB0_CNTL_MISC2__DISABLE_RDRET_CG_MASK |
 598                            DAGB0_CNTL_MISC2__DISABLE_TLBWR_CG_MASK |
 599                            DAGB0_CNTL_MISC2__DISABLE_TLBRD_CG_MASK);
 600
 601                 if (adev->asic_type != CHIP_RAVEN)
 602                         data2 &= ~(DAGB1_CNTL_MISC2__DISABLE_WRREQ_CG_MASK |
 603                                    DAGB1_CNTL_MISC2__DISABLE_WRRET_CG_MASK |
 604                                    DAGB1_CNTL_MISC2__DISABLE_RDREQ_CG_MASK |
 605                                    DAGB1_CNTL_MISC2__DISABLE_RDRET_CG_MASK |
 606                                    DAGB1_CNTL_MISC2__DISABLE_TLBWR_CG_MASK |
 607                                    DAGB1_CNTL_MISC2__DISABLE_TLBRD_CG_MASK);
 608         } else {
 609                 data &= ~ATC_L2_MISC_CG__ENABLE_MASK;
 610
 611                 data1 |= (DAGB0_CNTL_MISC2__DISABLE_WRREQ_CG_MASK |
 612                           DAGB0_CNTL_MISC2__DISABLE_WRRET_CG_MASK |
 613                           DAGB0_CNTL_MISC2__DISABLE_RDREQ_CG_MASK |
 614                           DAGB0_CNTL_MISC2__DISABLE_RDRET_CG_MASK |
 615                           DAGB0_CNTL_MISC2__DISABLE_TLBWR_CG_MASK |
 616                           DAGB0_CNTL_MISC2__DISABLE_TLBRD_CG_MASK);
 617
 618                 if (adev->asic_type != CHIP_RAVEN)
 619                         data2 |= (DAGB1_CNTL_MISC2__DISABLE_WRREQ_CG_MASK |
 620                                   DAGB1_CNTL_MISC2__DISABLE_WRRET_CG_MASK |
 621                                   DAGB1_CNTL_MISC2__DISABLE_RDREQ_CG_MASK |
 622                                   DAGB1_CNTL_MISC2__DISABLE_RDRET_CG_MASK |
 623                                   DAGB1_CNTL_MISC2__DISABLE_TLBWR_CG_MASK |
 624                                   DAGB1_CNTL_MISC2__DISABLE_TLBRD_CG_MASK);
 625         }
 626
 627         if (def != data)
 628                 WREG32_SOC15(MMHUB, 0, mmATC_L2_MISC_CG, data);
 629
 630         if (def1 != data1) {
 631                 if (adev->asic_type != CHIP_RAVEN)
 632                         WREG32_SOC15(MMHUB, 0, mmDAGB0_CNTL_MISC2, data1);
 633                 else
 634                         WREG32_SOC15(MMHUB, 0, mmDAGB0_CNTL_MISC2_RV, data1);
 635         }
 636
 637         if (adev->asic_type != CHIP_RAVEN && def2 != data2)
 638                 WREG32_SOC15(MMHUB, 0, mmDAGB1_CNTL_MISC2, data2);
 639 }
 640
 641 static void athub_update_medium_grain_clock_gating(struct amdgpu_device *adev,
 642                                                    bool enable)
 643 {
 644         uint32_t def, data;
 645
 646         def = data = RREG32_SOC15(ATHUB, 0, mmATHUB_MISC_CNTL);
 647
 648         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_MGCG))
 649                 data |= ATHUB_MISC_CNTL__CG_ENABLE_MASK;
 650         else
 651                 data &= ~ATHUB_MISC_CNTL__CG_ENABLE_MASK;
 652
 653         if (def != data)
 654                 WREG32_SOC15(ATHUB, 0, mmATHUB_MISC_CNTL, data);
 655 }
 656
 657 static void mmhub_v1_0_update_medium_grain_light_sleep(struct amdgpu_device *adev,
 658                                                        bool enable)
 659 {
 660         uint32_t def, data;
 661
 662         def = data = RREG32_SOC15(MMHUB, 0, mmATC_L2_MISC_CG);
 663
 664         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_LS))
 665                 data |= ATC_L2_MISC_CG__MEM_LS_ENABLE_MASK;
 666         else
 667                 data &= ~ATC_L2_MISC_CG__MEM_LS_ENABLE_MASK;
 668
 669         if (def != data)
 670                 WREG32_SOC15(MMHUB, 0, mmATC_L2_MISC_CG, data);
 671 }
 672
 673 static void athub_update_medium_grain_light_sleep(struct amdgpu_device *adev,
 674                                                   bool enable)
 675 {
 676         uint32_t def, data;
 677
 678         def = data = RREG32_SOC15(ATHUB, 0, mmATHUB_MISC_CNTL);
 679
 680         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_LS) &&
 681             (adev->cg_flags & AMD_CG_SUPPORT_HDP_LS))
 682                 data |= ATHUB_MISC_CNTL__CG_MEM_LS_ENABLE_MASK;
 683         else
 684                 data &= ~ATHUB_MISC_CNTL__CG_MEM_LS_ENABLE_MASK;
 685
 686         if(def != data)
 687                 WREG32_SOC15(ATHUB, 0, mmATHUB_MISC_CNTL, data);
 688 }
 689
 690 int mmhub_v1_0_set_clockgating(struct amdgpu_device *adev,
 691                                enum amd_clockgating_state state)
 692 {
 693         if (amdgpu_sriov_vf(adev))
 694                 return 0;
 695
 696         switch (adev->asic_type) {
 697         case CHIP_VEGA10:
 698         case CHIP_RAVEN:
 699                 mmhub_v1_0_update_medium_grain_clock_gating(adev,
 700                                 state == AMD_CG_STATE_GATE ? true : false);
 701                 athub_update_medium_grain_clock_gating(adev,
 702                                 state == AMD_CG_STATE_GATE ? true : false);
 703                 mmhub_v1_0_update_medium_grain_light_sleep(adev,
 704                                 state == AMD_CG_STATE_GATE ? true : false);
 705                 athub_update_medium_grain_light_sleep(adev,
 706                                 state == AMD_CG_STATE_GATE ? true : false);
 707                 break;
 708         default:
 709                 break;
 710         }
 711
 712         return 0;
 713 }
 714
 715 void mmhub_v1_0_get_clockgating(struct amdgpu_device *adev, u32 *flags)
 716 {
 717         int data;
 718
 719         if (amdgpu_sriov_vf(adev))
 720                 *flags = 0;
 721
 722         /* AMD_CG_SUPPORT_MC_MGCG */
 723         data = RREG32_SOC15(ATHUB, 0, mmATHUB_MISC_CNTL);
 724         if (data & ATHUB_MISC_CNTL__CG_ENABLE_MASK)
 725                 *flags |= AMD_CG_SUPPORT_MC_MGCG;
 726
 727         /* AMD_CG_SUPPORT_MC_LS */
 728         data = RREG32_SOC15(MMHUB, 0, mmATC_L2_MISC_CG);
 729         if (data & ATC_L2_MISC_CG__MEM_LS_ENABLE_MASK)
 730                 *flags |= AMD_CG_SUPPORT_MC_LS;
 731 }