drivers/gpu/drm/amd/amdgpu/mmhub_v1_0.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23 #include "amdgpu.h"
  24 #include "mmhub_v1_0.h"
  25
  26 #include "soc15ip.h"
  27 #include "mmhub/mmhub_1_0_offset.h"
  28 #include "mmhub/mmhub_1_0_sh_mask.h"
  29 #include "mmhub/mmhub_1_0_default.h"
  30 #include "athub/athub_1_0_offset.h"
  31 #include "athub/athub_1_0_sh_mask.h"
  32 #include "vega10_enum.h"
  33
  34 #include "soc15_common.h"
  35
  36 #define mmDAGB0_CNTL_MISC2_RV 0x008f
  37 #define mmDAGB0_CNTL_MISC2_RV_BASE_IDX 0
  38
  39 u64 mmhub_v1_0_get_fb_location(struct amdgpu_device *adev)
  40 {
  41         u64 base = RREG32_SOC15(MMHUB, 0, mmMC_VM_FB_LOCATION_BASE);
  42
  43         base &= MC_VM_FB_LOCATION_BASE__FB_BASE_MASK;
  44         base <<= 24;
  45
  46         return base;
  47 }
  48
  49 static void mmhub_v1_0_init_gart_pt_regs(struct amdgpu_device *adev)
  50 {
  51         uint64_t value;
  52
  53         BUG_ON(adev->gart.table_addr & (~0x0000FFFFFFFFF000ULL));
  54         value = adev->gart.table_addr - adev->mc.vram_start +
  55                 adev->vm_manager.vram_base_offset;
  56         value &= 0x0000FFFFFFFFF000ULL;
  57         value |= 0x1; /* valid bit */
  58
  59         WREG32_SOC15(MMHUB, 0, mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32,
  60                      lower_32_bits(value));
  61
  62         WREG32_SOC15(MMHUB, 0, mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32,
  63                      upper_32_bits(value));
  64 }
  65
  66 static void mmhub_v1_0_init_gart_aperture_regs(struct amdgpu_device *adev)
  67 {
  68         mmhub_v1_0_init_gart_pt_regs(adev);
  69
  70         WREG32_SOC15(MMHUB, 0, mmVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32,
  71                      (u32)(adev->mc.gart_start >> 12));
  72         WREG32_SOC15(MMHUB, 0, mmVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32,
  73                      (u32)(adev->mc.gart_start >> 44));
  74
  75         WREG32_SOC15(MMHUB, 0, mmVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32,
  76                      (u32)(adev->mc.gart_end >> 12));
  77         WREG32_SOC15(MMHUB, 0, mmVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32,
  78                      (u32)(adev->mc.gart_end >> 44));
  79 }
  80
  81 static void mmhub_v1_0_init_system_aperture_regs(struct amdgpu_device *adev)
  82 {
  83         uint64_t value;
  84         uint32_t tmp;
  85
  86         /* Disable AGP. */
  87         WREG32_SOC15(MMHUB, 0, mmMC_VM_AGP_BASE, 0);
  88         WREG32_SOC15(MMHUB, 0, mmMC_VM_AGP_TOP, 0);
  89         WREG32_SOC15(MMHUB, 0, mmMC_VM_AGP_BOT, 0x00FFFFFF);
  90
  91         /* Program the system aperture low logical page number. */
  92         WREG32_SOC15(MMHUB, 0, mmMC_VM_SYSTEM_APERTURE_LOW_ADDR,
  93                      adev->mc.vram_start >> 18);
  94         WREG32_SOC15(MMHUB, 0, mmMC_VM_SYSTEM_APERTURE_HIGH_ADDR,
  95                      adev->mc.vram_end >> 18);
  96
  97         /* Set default page address. */
  98         value = adev->vram_scratch.gpu_addr - adev->mc.vram_start +
  99                 adev->vm_manager.vram_base_offset;
 100         WREG32_SOC15(MMHUB, 0, mmMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB,
 101                      (u32)(value >> 12));
 102         WREG32_SOC15(MMHUB, 0, mmMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB,
 103                      (u32)(value >> 44));
 104
 105         /* Program "protection fault". */
 106         WREG32_SOC15(MMHUB, 0, mmVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32,
 107                      (u32)(adev->dummy_page.addr >> 12));
 108         WREG32_SOC15(MMHUB, 0, mmVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32,
 109                      (u32)((u64)adev->dummy_page.addr >> 44));
 110
 111         tmp = RREG32_SOC15(MMHUB, 0, mmVM_L2_PROTECTION_FAULT_CNTL2);
 112         tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL2,
 113                             ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1);
 114         WREG32_SOC15(MMHUB, 0, mmVM_L2_PROTECTION_FAULT_CNTL2, tmp);
 115 }
 116
 117 static void mmhub_v1_0_init_tlb_regs(struct amdgpu_device *adev)
 118 {
 119         uint32_t tmp;
 120
 121         /* Setup TLB control */
 122         tmp = RREG32_SOC15(MMHUB, 0, mmMC_VM_MX_L1_TLB_CNTL);
 123
 124         tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 1);
 125         tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE, 3);
 126         tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL,
 127                             ENABLE_ADVANCED_DRIVER_MODEL, 1);
 128         tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL,
 129                             SYSTEM_APERTURE_UNMAPPED_ACCESS, 0);
 130         tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ECO_BITS, 0);
 131         tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL,
 132                             MTYPE, MTYPE_UC);/* XXX for emulation. */
 133         tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ATC_EN, 1);
 134
 135         WREG32_SOC15(MMHUB, 0, mmMC_VM_MX_L1_TLB_CNTL, tmp);
 136 }
 137
 138 static void mmhub_v1_0_init_cache_regs(struct amdgpu_device *adev)
 139 {
 140         uint32_t tmp;
 141
 142         /* Setup L2 cache */
 143         tmp = RREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL);
 144         tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 1);
 145         tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING, 1);
 146         /* XXX for emulation, Refer to closed source code.*/
 147         tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, L2_PDE0_CACHE_TAG_GENERATION_MODE,
 148                             0);
 149         tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, PDE_FAULT_CLASSIFICATION, 1);
 150         tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, CONTEXT1_IDENTITY_ACCESS_MODE, 1);
 151         tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, IDENTITY_MODE_FRAGMENT_SIZE, 0);
 152         WREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL, tmp);
 153
 154         tmp = RREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL2);
 155         tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 1);
 156         tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_L2_CACHE, 1);
 157         WREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL2, tmp);
 158
 159         tmp = mmVM_L2_CNTL3_DEFAULT;
 160         tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 9);
 161         tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_FRAGMENT_SIZE, 6);
 162         WREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL3, tmp);
 163
 164         tmp = mmVM_L2_CNTL4_DEFAULT;
 165         tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_PDE_REQUEST_PHYSICAL, 0);
 166         tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_PTE_REQUEST_PHYSICAL, 0);
 167         WREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL4, tmp);
 168 }
 169
 170 static void mmhub_v1_0_enable_system_domain(struct amdgpu_device *adev)
 171 {
 172         uint32_t tmp;
 173
 174         tmp = RREG32_SOC15(MMHUB, 0, mmVM_CONTEXT0_CNTL);
 175         tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1);
 176         tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH, 0);
 177         WREG32_SOC15(MMHUB, 0, mmVM_CONTEXT0_CNTL, tmp);
 178 }
 179
 180 static void mmhub_v1_0_disable_identity_aperture(struct amdgpu_device *adev)
 181 {
 182         WREG32_SOC15(MMHUB, 0, mmVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32,
 183                      0XFFFFFFFF);
 184         WREG32_SOC15(MMHUB, 0, mmVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32,
 185                      0x0000000F);
 186
 187         WREG32_SOC15(MMHUB, 0,
 188                      mmVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32, 0);
 189         WREG32_SOC15(MMHUB, 0,
 190                      mmVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32, 0);
 191
 192         WREG32_SOC15(MMHUB, 0, mmVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32,
 193                      0);
 194         WREG32_SOC15(MMHUB, 0, mmVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32,
 195                      0);
 196 }
 197
 198 static void mmhub_v1_0_setup_vmid_config(struct amdgpu_device *adev)
 199 {
 200         int i;
 201         uint32_t tmp;
 202
 203         for (i = 0; i <= 14; i++) {
 204                 tmp = RREG32_SOC15_OFFSET(MMHUB, 0, mmVM_CONTEXT1_CNTL, i);
 205                 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 206                                 ENABLE_CONTEXT, 1);
 207                 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 208                                 PAGE_TABLE_DEPTH, adev->vm_manager.num_level);
 209                 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 210                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 211                 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 212                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 213                 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 214                                 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 215                 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 216                                 VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 217                 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 218                                 READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 219                 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 220                                 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 221                 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 222                                 EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
 223                 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 224                                 PAGE_TABLE_BLOCK_SIZE,
 225                                 adev->vm_manager.block_size - 9);
 226                 /* Send no-retry XNACK on fault to suppress VM fault storm. */
 227                 tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 228                                     RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 0);
 229                 WREG32_SOC15_OFFSET(MMHUB, 0, mmVM_CONTEXT1_CNTL, i, tmp);
 230                 WREG32_SOC15_OFFSET(MMHUB, 0, mmVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32, i*2, 0);
 231                 WREG32_SOC15_OFFSET(MMHUB, 0, mmVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32, i*2, 0);
 232                 WREG32_SOC15_OFFSET(MMHUB, 0, mmVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32, i*2,
 233                         lower_32_bits(adev->vm_manager.max_pfn - 1));
 234                 WREG32_SOC15_OFFSET(MMHUB, 0, mmVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32, i*2,
 235                         upper_32_bits(adev->vm_manager.max_pfn - 1));
 236         }
 237 }
 238
 239 static void mmhub_v1_0_program_invalidation(struct amdgpu_device *adev)
 240 {
 241         unsigned i;
 242
 243         for (i = 0; i < 18; ++i) {
 244                 WREG32_SOC15_OFFSET(MMHUB, 0, mmVM_INVALIDATE_ENG0_ADDR_RANGE_LO32,
 245                                     2 * i, 0xffffffff);
 246                 WREG32_SOC15_OFFSET(MMHUB, 0, mmVM_INVALIDATE_ENG0_ADDR_RANGE_HI32,
 247                                     2 * i, 0x1f);
 248         }
 249 }
 250
 251 struct pctl_data {
 252         uint32_t index;
 253         uint32_t data;
 254 };
 255
 256 static const struct pctl_data pctl0_data[] = {
 257         {0x0, 0x7a640},
 258         {0x9, 0x2a64a},
 259         {0xd, 0x2a680},
 260         {0x11, 0x6a684},
 261         {0x19, 0xea68e},
 262         {0x29, 0xa69e},
 263         {0x2b, 0x34a6c0},
 264         {0x61, 0x83a707},
 265         {0xe6, 0x8a7a4},
 266         {0xf0, 0x1a7b8},
 267         {0xf3, 0xfa7cc},
 268         {0x104, 0x17a7dd},
 269         {0x11d, 0xa7dc},
 270         {0x11f, 0x12a7f5},
 271         {0x133, 0xa808},
 272         {0x135, 0x12a810},
 273         {0x149, 0x7a82c}
 274 };
 275 #define PCTL0_DATA_LEN (ARRAY_SIZE(pctl0_data))
 276
 277 #define PCTL0_RENG_EXEC_END_PTR 0x151
 278 #define PCTL0_STCTRL_REG_SAVE_RANGE0_BASE  0xa640
 279 #define PCTL0_STCTRL_REG_SAVE_RANGE0_LIMIT 0xa833
 280
 281 static const struct pctl_data pctl1_data[] = {
 282         {0x0, 0x39a000},
 283         {0x3b, 0x44a040},
 284         {0x81, 0x2a08d},
 285         {0x85, 0x6ba094},
 286         {0xf2, 0x18a100},
 287         {0x10c, 0x4a132},
 288         {0x112, 0xca141},
 289         {0x120, 0x2fa158},
 290         {0x151, 0x17a1d0},
 291         {0x16a, 0x1a1e9},
 292         {0x16d, 0x13a1ec},
 293         {0x182, 0x7a201},
 294         {0x18b, 0x3a20a},
 295         {0x190, 0x7a580},
 296         {0x199, 0xa590},
 297         {0x19b, 0x4a594},
 298         {0x1a1, 0x1a59c},
 299         {0x1a4, 0x7a82c},
 300         {0x1ad, 0xfa7cc},
 301         {0x1be, 0x17a7dd},
 302         {0x1d7, 0x12a810},
 303         {0x1eb, 0x4000a7e1},
 304         {0x1ec, 0x5000a7f5},
 305         {0x1ed, 0x4000a7e2},
 306         {0x1ee, 0x5000a7dc},
 307         {0x1ef, 0x4000a7e3},
 308         {0x1f0, 0x5000a7f6},
 309         {0x1f1, 0x5000a7e4}
 310 };
 311 #define PCTL1_DATA_LEN (ARRAY_SIZE(pctl1_data))
 312
 313 #define PCTL1_RENG_EXEC_END_PTR 0x1f1
 314 #define PCTL1_STCTRL_REG_SAVE_RANGE0_BASE  0xa000
 315 #define PCTL1_STCTRL_REG_SAVE_RANGE0_LIMIT 0xa20d
 316 #define PCTL1_STCTRL_REG_SAVE_RANGE1_BASE  0xa580
 317 #define PCTL1_STCTRL_REG_SAVE_RANGE1_LIMIT 0xa59d
 318 #define PCTL1_STCTRL_REG_SAVE_RANGE2_BASE  0xa82c
 319 #define PCTL1_STCTRL_REG_SAVE_RANGE2_LIMIT 0xa833
 320
 321 static void mmhub_v1_0_power_gating_write_save_ranges(struct amdgpu_device *adev)
 322 {
 323         uint32_t tmp = 0;
 324
 325         /* PCTL0_STCTRL_REGISTER_SAVE_RANGE0 */
 326         tmp = REG_SET_FIELD(tmp, PCTL0_STCTRL_REGISTER_SAVE_RANGE0,
 327                         STCTRL_REGISTER_SAVE_BASE,
 328                         PCTL0_STCTRL_REG_SAVE_RANGE0_BASE);
 329         tmp = REG_SET_FIELD(tmp, PCTL0_STCTRL_REGISTER_SAVE_RANGE0,
 330                         STCTRL_REGISTER_SAVE_LIMIT,
 331                         PCTL0_STCTRL_REG_SAVE_RANGE0_LIMIT);
 332         WREG32_SOC15(MMHUB, 0, mmPCTL0_STCTRL_REGISTER_SAVE_RANGE0, tmp);
 333
 334         /* PCTL1_STCTRL_REGISTER_SAVE_RANGE0 */
 335         tmp = 0;
 336         tmp = REG_SET_FIELD(tmp, PCTL1_STCTRL_REGISTER_SAVE_RANGE0,
 337                         STCTRL_REGISTER_SAVE_BASE,
 338                         PCTL1_STCTRL_REG_SAVE_RANGE0_BASE);
 339         tmp = REG_SET_FIELD(tmp, PCTL1_STCTRL_REGISTER_SAVE_RANGE0,
 340                         STCTRL_REGISTER_SAVE_LIMIT,
 341                         PCTL1_STCTRL_REG_SAVE_RANGE0_LIMIT);
 342         WREG32_SOC15(MMHUB, 0, mmPCTL1_STCTRL_REGISTER_SAVE_RANGE0, tmp);
 343
 344         /* PCTL1_STCTRL_REGISTER_SAVE_RANGE1 */
 345         tmp = 0;
 346         tmp = REG_SET_FIELD(tmp, PCTL1_STCTRL_REGISTER_SAVE_RANGE1,
 347                         STCTRL_REGISTER_SAVE_BASE,
 348                         PCTL1_STCTRL_REG_SAVE_RANGE1_BASE);
 349         tmp = REG_SET_FIELD(tmp, PCTL1_STCTRL_REGISTER_SAVE_RANGE1,
 350                         STCTRL_REGISTER_SAVE_LIMIT,
 351                         PCTL1_STCTRL_REG_SAVE_RANGE1_LIMIT);
 352         WREG32_SOC15(MMHUB, 0, mmPCTL1_STCTRL_REGISTER_SAVE_RANGE1, tmp);
 353
 354         /* PCTL1_STCTRL_REGISTER_SAVE_RANGE2 */
 355         tmp = 0;
 356         tmp = REG_SET_FIELD(tmp, PCTL1_STCTRL_REGISTER_SAVE_RANGE2,
 357                         STCTRL_REGISTER_SAVE_BASE,
 358                         PCTL1_STCTRL_REG_SAVE_RANGE2_BASE);
 359         tmp = REG_SET_FIELD(tmp, PCTL1_STCTRL_REGISTER_SAVE_RANGE2,
 360                         STCTRL_REGISTER_SAVE_LIMIT,
 361                         PCTL1_STCTRL_REG_SAVE_RANGE2_LIMIT);
 362         WREG32_SOC15(MMHUB, 0, mmPCTL1_STCTRL_REGISTER_SAVE_RANGE2, tmp);
 363 }
 364
 365 void mmhub_v1_0_initialize_power_gating(struct amdgpu_device *adev)
 366 {
 367         uint32_t pctl0_misc = 0;
 368         uint32_t pctl0_reng_execute = 0;
 369         uint32_t pctl1_misc = 0;
 370         uint32_t pctl1_reng_execute = 0;
 371         int i = 0;
 372
 373         if (amdgpu_sriov_vf(adev))
 374                 return;
 375
 376         pctl0_misc = RREG32_SOC15(MMHUB, 0, mmPCTL0_MISC);
 377         pctl0_reng_execute = RREG32_SOC15(MMHUB, 0, mmPCTL0_RENG_EXECUTE);
 378         pctl1_misc = RREG32_SOC15(MMHUB, 0, mmPCTL1_MISC);
 379         pctl1_reng_execute = RREG32_SOC15(MMHUB, 0, mmPCTL1_RENG_EXECUTE);
 380
 381         /* Light sleep must be disabled before writing to pctl0 registers */
 382         pctl0_misc &= ~PCTL0_MISC__RENG_MEM_LS_ENABLE_MASK;
 383         WREG32_SOC15(MMHUB, 0, mmPCTL0_MISC, pctl0_misc);
 384
 385         /* Write data used to access ram of register engine */
 386         for (i = 0; i < PCTL0_DATA_LEN; i++) {
 387                 WREG32_SOC15(MMHUB, 0, mmPCTL0_RENG_RAM_INDEX,
 388                         pctl0_data[i].index);
 389                 WREG32_SOC15(MMHUB, 0, mmPCTL0_RENG_RAM_DATA,
 390                         pctl0_data[i].data);
 391         }
 392
 393         /* Set the reng execute end ptr for pctl0 */
 394         pctl0_reng_execute = REG_SET_FIELD(pctl0_reng_execute,
 395                                         PCTL0_RENG_EXECUTE,
 396                                         RENG_EXECUTE_END_PTR,
 397                                         PCTL0_RENG_EXEC_END_PTR);
 398         WREG32_SOC15(MMHUB, 0, mmPCTL0_RENG_EXECUTE, pctl0_reng_execute);
 399
 400         /* Light sleep must be disabled before writing to pctl1 registers */
 401         pctl1_misc &= ~PCTL1_MISC__RENG_MEM_LS_ENABLE_MASK;
 402         WREG32_SOC15(MMHUB, 0, mmPCTL1_MISC, pctl1_misc);
 403
 404         /* Write data used to access ram of register engine */
 405         for (i = 0; i < PCTL1_DATA_LEN; i++) {
 406                 WREG32_SOC15(MMHUB, 0, mmPCTL1_RENG_RAM_INDEX,
 407                         pctl1_data[i].index);
 408                 WREG32_SOC15(MMHUB, 0, mmPCTL1_RENG_RAM_DATA,
 409                         pctl1_data[i].data);
 410         }
 411
 412         /* Set the reng execute end ptr for pctl1 */
 413         pctl1_reng_execute = REG_SET_FIELD(pctl1_reng_execute,
 414                                         PCTL1_RENG_EXECUTE,
 415                                         RENG_EXECUTE_END_PTR,
 416                                         PCTL1_RENG_EXEC_END_PTR);
 417         WREG32_SOC15(MMHUB, 0, mmPCTL1_RENG_EXECUTE, pctl1_reng_execute);
 418
 419         mmhub_v1_0_power_gating_write_save_ranges(adev);
 420
 421         /* Re-enable light sleep */
 422         pctl0_misc |= PCTL0_MISC__RENG_MEM_LS_ENABLE_MASK;
 423         WREG32_SOC15(MMHUB, 0, mmPCTL0_MISC, pctl0_misc);
 424         pctl1_misc |= PCTL1_MISC__RENG_MEM_LS_ENABLE_MASK;
 425         WREG32_SOC15(MMHUB, 0, mmPCTL1_MISC, pctl1_misc);
 426 }
 427
 428 void mmhub_v1_0_update_power_gating(struct amdgpu_device *adev,
 429                                 bool enable)
 430 {
 431         uint32_t pctl0_reng_execute = 0;
 432         uint32_t pctl1_reng_execute = 0;
 433
 434         if (amdgpu_sriov_vf(adev))
 435                 return;
 436
 437         pctl0_reng_execute = RREG32_SOC15(MMHUB, 0, mmPCTL0_RENG_EXECUTE);
 438         pctl1_reng_execute = RREG32_SOC15(MMHUB, 0, mmPCTL1_RENG_EXECUTE);
 439
 440         if (enable && adev->pg_flags & AMD_PG_SUPPORT_MMHUB) {
 441                 pctl0_reng_execute = REG_SET_FIELD(pctl0_reng_execute,
 442                                                 PCTL0_RENG_EXECUTE,
 443                                                 RENG_EXECUTE_ON_PWR_UP, 1);
 444                 pctl0_reng_execute = REG_SET_FIELD(pctl0_reng_execute,
 445                                                 PCTL0_RENG_EXECUTE,
 446                                                 RENG_EXECUTE_ON_REG_UPDATE, 1);
 447                 WREG32_SOC15(MMHUB, 0, mmPCTL0_RENG_EXECUTE, pctl0_reng_execute);
 448
 449                 pctl1_reng_execute = REG_SET_FIELD(pctl1_reng_execute,
 450                                                 PCTL1_RENG_EXECUTE,
 451                                                 RENG_EXECUTE_ON_PWR_UP, 1);
 452                 pctl1_reng_execute = REG_SET_FIELD(pctl1_reng_execute,
 453                                                 PCTL1_RENG_EXECUTE,
 454                                                 RENG_EXECUTE_ON_REG_UPDATE, 1);
 455                 WREG32_SOC15(MMHUB, 0, mmPCTL1_RENG_EXECUTE, pctl1_reng_execute);
 456
 457         } else {
 458                 pctl0_reng_execute = REG_SET_FIELD(pctl0_reng_execute,
 459                                                 PCTL0_RENG_EXECUTE,
 460                                                 RENG_EXECUTE_ON_PWR_UP, 0);
 461                 pctl0_reng_execute = REG_SET_FIELD(pctl0_reng_execute,
 462                                                 PCTL0_RENG_EXECUTE,
 463                                                 RENG_EXECUTE_ON_REG_UPDATE, 0);
 464                 WREG32_SOC15(MMHUB, 0, mmPCTL0_RENG_EXECUTE, pctl0_reng_execute);
 465
 466                 pctl1_reng_execute = REG_SET_FIELD(pctl1_reng_execute,
 467                                                 PCTL1_RENG_EXECUTE,
 468                                                 RENG_EXECUTE_ON_PWR_UP, 0);
 469                 pctl1_reng_execute = REG_SET_FIELD(pctl1_reng_execute,
 470                                                 PCTL1_RENG_EXECUTE,
 471                                                 RENG_EXECUTE_ON_REG_UPDATE, 0);
 472                 WREG32_SOC15(MMHUB, 0, mmPCTL1_RENG_EXECUTE, pctl1_reng_execute);
 473         }
 474 }
 475
 476 int mmhub_v1_0_gart_enable(struct amdgpu_device *adev)
 477 {
 478         if (amdgpu_sriov_vf(adev)) {
 479                 /*
 480                  * MC_VM_FB_LOCATION_BASE/TOP is NULL for VF, becuase they are
 481                  * VF copy registers so vbios post doesn't program them, for
 482                  * SRIOV driver need to program them
 483                  */
 484                 WREG32_SOC15(MMHUB, 0, mmMC_VM_FB_LOCATION_BASE,
 485                              adev->mc.vram_start >> 24);
 486                 WREG32_SOC15(MMHUB, 0, mmMC_VM_FB_LOCATION_TOP,
 487                              adev->mc.vram_end >> 24);
 488         }
 489
 490         /* GART Enable. */
 491         mmhub_v1_0_init_gart_aperture_regs(adev);
 492         mmhub_v1_0_init_system_aperture_regs(adev);
 493         mmhub_v1_0_init_tlb_regs(adev);
 494         mmhub_v1_0_init_cache_regs(adev);
 495
 496         mmhub_v1_0_enable_system_domain(adev);
 497         mmhub_v1_0_disable_identity_aperture(adev);
 498         mmhub_v1_0_setup_vmid_config(adev);
 499         mmhub_v1_0_program_invalidation(adev);
 500
 501         return 0;
 502 }
 503
 504 void mmhub_v1_0_gart_disable(struct amdgpu_device *adev)
 505 {
 506         u32 tmp;
 507         u32 i;
 508
 509         /* Disable all tables */
 510         for (i = 0; i < 16; i++)
 511                 WREG32_SOC15_OFFSET(MMHUB, 0, mmVM_CONTEXT0_CNTL, i, 0);
 512
 513         /* Setup TLB control */
 514         tmp = RREG32_SOC15(MMHUB, 0, mmMC_VM_MX_L1_TLB_CNTL);
 515         tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 0);
 516         tmp = REG_SET_FIELD(tmp,
 517                                 MC_VM_MX_L1_TLB_CNTL,
 518                                 ENABLE_ADVANCED_DRIVER_MODEL,
 519                                 0);
 520         WREG32_SOC15(MMHUB, 0, mmMC_VM_MX_L1_TLB_CNTL, tmp);
 521
 522         /* Setup L2 cache */
 523         tmp = RREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL);
 524         tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 0);
 525         WREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL, tmp);
 526         WREG32_SOC15(MMHUB, 0, mmVM_L2_CNTL3, 0);
 527 }
 528
 529 /**
 530  * mmhub_v1_0_set_fault_enable_default - update GART/VM fault handling
 531  *
 532  * @adev: amdgpu_device pointer
 533  * @value: true redirects VM faults to the default page
 534  */
 535 void mmhub_v1_0_set_fault_enable_default(struct amdgpu_device *adev, bool value)
 536 {
 537         u32 tmp;
 538         tmp = RREG32_SOC15(MMHUB, 0, mmVM_L2_PROTECTION_FAULT_CNTL);
 539         tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
 540                         RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
 541         tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
 542                         PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value);
 543         tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
 544                         PDE1_PROTECTION_FAULT_ENABLE_DEFAULT, value);
 545         tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
 546                         PDE2_PROTECTION_FAULT_ENABLE_DEFAULT, value);
 547         tmp = REG_SET_FIELD(tmp,
 548                         VM_L2_PROTECTION_FAULT_CNTL,
 549                         TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT,
 550                         value);
 551         tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
 552                         NACK_PROTECTION_FAULT_ENABLE_DEFAULT, value);
 553         tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
 554                         DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
 555         tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
 556                         VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value);
 557         tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
 558                         READ_PROTECTION_FAULT_ENABLE_DEFAULT, value);
 559         tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
 560                         WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
 561         tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
 562                         EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
 563         if (!value) {
 564                 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
 565                                 CRASH_ON_NO_RETRY_FAULT, 1);
 566                 tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
 567                                 CRASH_ON_RETRY_FAULT, 1);
 568     }
 569
 570         WREG32_SOC15(MMHUB, 0, mmVM_L2_PROTECTION_FAULT_CNTL, tmp);
 571 }
 572
 573 void mmhub_v1_0_init(struct amdgpu_device *adev)
 574 {
 575         struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB];
 576
 577         hub->ctx0_ptb_addr_lo32 =
 578                 SOC15_REG_OFFSET(MMHUB, 0,
 579                                  mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32);
 580         hub->ctx0_ptb_addr_hi32 =
 581                 SOC15_REG_OFFSET(MMHUB, 0,
 582                                  mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32);
 583         hub->vm_inv_eng0_req =
 584                 SOC15_REG_OFFSET(MMHUB, 0, mmVM_INVALIDATE_ENG0_REQ);
 585         hub->vm_inv_eng0_ack =
 586                 SOC15_REG_OFFSET(MMHUB, 0, mmVM_INVALIDATE_ENG0_ACK);
 587         hub->vm_context0_cntl =
 588                 SOC15_REG_OFFSET(MMHUB, 0, mmVM_CONTEXT0_CNTL);
 589         hub->vm_l2_pro_fault_status =
 590                 SOC15_REG_OFFSET(MMHUB, 0, mmVM_L2_PROTECTION_FAULT_STATUS);
 591         hub->vm_l2_pro_fault_cntl =
 592                 SOC15_REG_OFFSET(MMHUB, 0, mmVM_L2_PROTECTION_FAULT_CNTL);
 593
 594 }
 595
 596 static void mmhub_v1_0_update_medium_grain_clock_gating(struct amdgpu_device *adev,
 597                                                         bool enable)
 598 {
 599         uint32_t def, data, def1, data1, def2 = 0, data2 = 0;
 600
 601         def  = data  = RREG32_SOC15(MMHUB, 0, mmATC_L2_MISC_CG);
 602
 603         if (adev->asic_type != CHIP_RAVEN) {
 604                 def1 = data1 = RREG32_SOC15(MMHUB, 0, mmDAGB0_CNTL_MISC2);
 605                 def2 = data2 = RREG32_SOC15(MMHUB, 0, mmDAGB1_CNTL_MISC2);
 606         } else
 607                 def1 = data1 = RREG32_SOC15(MMHUB, 0, mmDAGB0_CNTL_MISC2_RV);
 608
 609         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_MGCG)) {
 610                 data |= ATC_L2_MISC_CG__ENABLE_MASK;
 611
 612                 data1 &= ~(DAGB0_CNTL_MISC2__DISABLE_WRREQ_CG_MASK |
 613                            DAGB0_CNTL_MISC2__DISABLE_WRRET_CG_MASK |
 614                            DAGB0_CNTL_MISC2__DISABLE_RDREQ_CG_MASK |
 615                            DAGB0_CNTL_MISC2__DISABLE_RDRET_CG_MASK |
 616                            DAGB0_CNTL_MISC2__DISABLE_TLBWR_CG_MASK |
 617                            DAGB0_CNTL_MISC2__DISABLE_TLBRD_CG_MASK);
 618
 619                 if (adev->asic_type != CHIP_RAVEN)
 620                         data2 &= ~(DAGB1_CNTL_MISC2__DISABLE_WRREQ_CG_MASK |
 621                                    DAGB1_CNTL_MISC2__DISABLE_WRRET_CG_MASK |
 622                                    DAGB1_CNTL_MISC2__DISABLE_RDREQ_CG_MASK |
 623                                    DAGB1_CNTL_MISC2__DISABLE_RDRET_CG_MASK |
 624                                    DAGB1_CNTL_MISC2__DISABLE_TLBWR_CG_MASK |
 625                                    DAGB1_CNTL_MISC2__DISABLE_TLBRD_CG_MASK);
 626         } else {
 627                 data &= ~ATC_L2_MISC_CG__ENABLE_MASK;
 628
 629                 data1 |= (DAGB0_CNTL_MISC2__DISABLE_WRREQ_CG_MASK |
 630                           DAGB0_CNTL_MISC2__DISABLE_WRRET_CG_MASK |
 631                           DAGB0_CNTL_MISC2__DISABLE_RDREQ_CG_MASK |
 632                           DAGB0_CNTL_MISC2__DISABLE_RDRET_CG_MASK |
 633                           DAGB0_CNTL_MISC2__DISABLE_TLBWR_CG_MASK |
 634                           DAGB0_CNTL_MISC2__DISABLE_TLBRD_CG_MASK);
 635
 636                 if (adev->asic_type != CHIP_RAVEN)
 637                         data2 |= (DAGB1_CNTL_MISC2__DISABLE_WRREQ_CG_MASK |
 638                                   DAGB1_CNTL_MISC2__DISABLE_WRRET_CG_MASK |
 639                                   DAGB1_CNTL_MISC2__DISABLE_RDREQ_CG_MASK |
 640                                   DAGB1_CNTL_MISC2__DISABLE_RDRET_CG_MASK |
 641                                   DAGB1_CNTL_MISC2__DISABLE_TLBWR_CG_MASK |
 642                                   DAGB1_CNTL_MISC2__DISABLE_TLBRD_CG_MASK);
 643         }
 644
 645         if (def != data)
 646                 WREG32_SOC15(MMHUB, 0, mmATC_L2_MISC_CG, data);
 647
 648         if (def1 != data1) {
 649                 if (adev->asic_type != CHIP_RAVEN)
 650                         WREG32_SOC15(MMHUB, 0, mmDAGB0_CNTL_MISC2, data1);
 651                 else
 652                         WREG32_SOC15(MMHUB, 0, mmDAGB0_CNTL_MISC2_RV, data1);
 653         }
 654
 655         if (adev->asic_type != CHIP_RAVEN && def2 != data2)
 656                 WREG32_SOC15(MMHUB, 0, mmDAGB1_CNTL_MISC2, data2);
 657 }
 658
 659 static void athub_update_medium_grain_clock_gating(struct amdgpu_device *adev,
 660                                                    bool enable)
 661 {
 662         uint32_t def, data;
 663
 664         def = data = RREG32_SOC15(ATHUB, 0, mmATHUB_MISC_CNTL);
 665
 666         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_MGCG))
 667                 data |= ATHUB_MISC_CNTL__CG_ENABLE_MASK;
 668         else
 669                 data &= ~ATHUB_MISC_CNTL__CG_ENABLE_MASK;
 670
 671         if (def != data)
 672                 WREG32_SOC15(ATHUB, 0, mmATHUB_MISC_CNTL, data);
 673 }
 674
 675 static void mmhub_v1_0_update_medium_grain_light_sleep(struct amdgpu_device *adev,
 676                                                        bool enable)
 677 {
 678         uint32_t def, data;
 679
 680         def = data = RREG32_SOC15(MMHUB, 0, mmATC_L2_MISC_CG);
 681
 682         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_LS))
 683                 data |= ATC_L2_MISC_CG__MEM_LS_ENABLE_MASK;
 684         else
 685                 data &= ~ATC_L2_MISC_CG__MEM_LS_ENABLE_MASK;
 686
 687         if (def != data)
 688                 WREG32_SOC15(MMHUB, 0, mmATC_L2_MISC_CG, data);
 689 }
 690
 691 static void athub_update_medium_grain_light_sleep(struct amdgpu_device *adev,
 692                                                   bool enable)
 693 {
 694         uint32_t def, data;
 695
 696         def = data = RREG32_SOC15(ATHUB, 0, mmATHUB_MISC_CNTL);
 697
 698         if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_LS) &&
 699             (adev->cg_flags & AMD_CG_SUPPORT_HDP_LS))
 700                 data |= ATHUB_MISC_CNTL__CG_MEM_LS_ENABLE_MASK;
 701         else
 702                 data &= ~ATHUB_MISC_CNTL__CG_MEM_LS_ENABLE_MASK;
 703
 704         if(def != data)
 705                 WREG32_SOC15(ATHUB, 0, mmATHUB_MISC_CNTL, data);
 706 }
 707
 708 int mmhub_v1_0_set_clockgating(struct amdgpu_device *adev,
 709                                enum amd_clockgating_state state)
 710 {
 711         if (amdgpu_sriov_vf(adev))
 712                 return 0;
 713
 714         switch (adev->asic_type) {
 715         case CHIP_VEGA10:
 716         case CHIP_RAVEN:
 717                 mmhub_v1_0_update_medium_grain_clock_gating(adev,
 718                                 state == AMD_CG_STATE_GATE ? true : false);
 719                 athub_update_medium_grain_clock_gating(adev,
 720                                 state == AMD_CG_STATE_GATE ? true : false);
 721                 mmhub_v1_0_update_medium_grain_light_sleep(adev,
 722                                 state == AMD_CG_STATE_GATE ? true : false);
 723                 athub_update_medium_grain_light_sleep(adev,
 724                                 state == AMD_CG_STATE_GATE ? true : false);
 725                 break;
 726         default:
 727                 break;
 728         }
 729
 730         return 0;
 731 }
 732
 733 void mmhub_v1_0_get_clockgating(struct amdgpu_device *adev, u32 *flags)
 734 {
 735         int data;
 736
 737         if (amdgpu_sriov_vf(adev))
 738                 *flags = 0;
 739
 740         /* AMD_CG_SUPPORT_MC_MGCG */
 741         data = RREG32_SOC15(ATHUB, 0, mmATHUB_MISC_CNTL);
 742         if (data & ATHUB_MISC_CNTL__CG_ENABLE_MASK)
 743                 *flags |= AMD_CG_SUPPORT_MC_MGCG;
 744
 745         /* AMD_CG_SUPPORT_MC_LS */
 746         data = RREG32_SOC15(MMHUB, 0, mmATC_L2_MISC_CG);
 747         if (data & ATC_L2_MISC_CG__MEM_LS_ENABLE_MASK)
 748                 *flags |= AMD_CG_SUPPORT_MC_LS;
 749 }