drivers/platform/x86/intel/pmc/core.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Intel Core SoC Power Management Controller Driver
   4  *
   5  * Copyright (c) 2016, Intel Corporation.
   6  * All Rights Reserved.
   7  *
   8  * Authors: Rajneesh Bhardwaj <[email protected]>
   9  *          Vishwanath Somayaji <[email protected]>
  10  */
  11
  12 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  13
  14 #include <linux/bitfield.h>
  15 #include <linux/debugfs.h>
  16 #include <linux/delay.h>
  17 #include <linux/dmi.h>
  18 #include <linux/io.h>
  19 #include <linux/module.h>
  20 #include <linux/pci.h>
  21 #include <linux/slab.h>
  22 #include <linux/suspend.h>
  23 #include <linux/units.h>
  24
  25 #include <asm/cpu_device_id.h>
  26 #include <asm/intel-family.h>
  27 #include <asm/msr.h>
  28 #include <asm/tsc.h>
  29
  30 #include "core.h"
  31 #include "../pmt/telemetry.h"
  32
  33 /* Maximum number of modes supported by platfoms that has low power mode capability */
  34 const char *pmc_lpm_modes[] = {
  35         "S0i2.0",
  36         "S0i2.1",
  37         "S0i2.2",
  38         "S0i3.0",
  39         "S0i3.1",
  40         "S0i3.2",
  41         "S0i3.3",
  42         "S0i3.4",
  43         NULL
  44 };
  45
  46 /* PKGC MSRs are common across Intel Core SoCs */
  47 const struct pmc_bit_map msr_map[] = {
  48         {"Package C2",                  MSR_PKG_C2_RESIDENCY},
  49         {"Package C3",                  MSR_PKG_C3_RESIDENCY},
  50         {"Package C6",                  MSR_PKG_C6_RESIDENCY},
  51         {"Package C7",                  MSR_PKG_C7_RESIDENCY},
  52         {"Package C8",                  MSR_PKG_C8_RESIDENCY},
  53         {"Package C9",                  MSR_PKG_C9_RESIDENCY},
  54         {"Package C10",                 MSR_PKG_C10_RESIDENCY},
  55         {}
  56 };
  57
  58 static inline u32 pmc_core_reg_read(struct pmc *pmc, int reg_offset)
  59 {
  60         return readl(pmc->regbase + reg_offset);
  61 }
  62
  63 static inline void pmc_core_reg_write(struct pmc *pmc, int reg_offset,
  64                                       u32 val)
  65 {
  66         writel(val, pmc->regbase + reg_offset);
  67 }
  68
  69 static inline u64 pmc_core_adjust_slp_s0_step(struct pmc *pmc, u32 value)
  70 {
  71         /*
  72          * ADL PCH does not have the SLP_S0 counter and LPM Residency counters are
  73          * used as a workaround which uses 30.5 usec tick. All other client
  74          * programs have the legacy SLP_S0 residency counter that is using the 122
  75          * usec tick.
  76          */
  77         const int lpm_adj_x2 = pmc->map->lpm_res_counter_step_x2;
  78
  79         if (pmc->map == &adl_reg_map)
  80                 return (u64)value * GET_X2_COUNTER((u64)lpm_adj_x2);
  81         else
  82                 return (u64)value * pmc->map->slp_s0_res_counter_step;
  83 }
  84
  85 static int set_etr3(struct pmc_dev *pmcdev)
  86 {
  87         struct pmc *pmc = pmcdev->pmcs[PMC_IDX_MAIN];
  88         const struct pmc_reg_map *map = pmc->map;
  89         u32 reg;
  90
  91         if (!map->etr3_offset)
  92                 return -EOPNOTSUPP;
  93
  94         guard(mutex)(&pmcdev->lock);
  95
  96         /* check if CF9 is locked */
  97         reg = pmc_core_reg_read(pmc, map->etr3_offset);
  98         if (reg & ETR3_CF9LOCK)
  99                 return -EACCES;
 100
 101         /* write CF9 global reset bit */
 102         reg |= ETR3_CF9GR;
 103         pmc_core_reg_write(pmc, map->etr3_offset, reg);
 104
 105         reg = pmc_core_reg_read(pmc, map->etr3_offset);
 106         if (!(reg & ETR3_CF9GR))
 107                 return -EIO;
 108
 109         return 0;
 110 }
 111 static umode_t etr3_is_visible(struct kobject *kobj,
 112                                 struct attribute *attr,
 113                                 int idx)
 114 {
 115         struct device *dev = kobj_to_dev(kobj);
 116         struct pmc_dev *pmcdev = dev_get_drvdata(dev);
 117         struct pmc *pmc = pmcdev->pmcs[PMC_IDX_MAIN];
 118         const struct pmc_reg_map *map = pmc->map;
 119         u32 reg;
 120
 121         scoped_guard(mutex, &pmcdev->lock)
 122                 reg = pmc_core_reg_read(pmc, map->etr3_offset);
 123
 124         return reg & ETR3_CF9LOCK ? attr->mode & (SYSFS_PREALLOC | 0444) : attr->mode;
 125 }
 126
 127 static ssize_t etr3_show(struct device *dev,
 128                                  struct device_attribute *attr, char *buf)
 129 {
 130         struct pmc_dev *pmcdev = dev_get_drvdata(dev);
 131         struct pmc *pmc = pmcdev->pmcs[PMC_IDX_MAIN];
 132         const struct pmc_reg_map *map = pmc->map;
 133         u32 reg;
 134
 135         if (!map->etr3_offset)
 136                 return -EOPNOTSUPP;
 137
 138         scoped_guard(mutex, &pmcdev->lock) {
 139                 reg = pmc_core_reg_read(pmc, map->etr3_offset);
 140                 reg &= ETR3_CF9GR | ETR3_CF9LOCK;
 141         }
 142
 143         return sysfs_emit(buf, "0x%08x", reg);
 144 }
 145
 146 static ssize_t etr3_store(struct device *dev,
 147                                   struct device_attribute *attr,
 148                                   const char *buf, size_t len)
 149 {
 150         struct pmc_dev *pmcdev = dev_get_drvdata(dev);
 151         int err;
 152         u32 reg;
 153
 154         err = kstrtouint(buf, 16, &reg);
 155         if (err)
 156                 return err;
 157
 158         /* allow only CF9 writes */
 159         if (reg != ETR3_CF9GR)
 160                 return -EINVAL;
 161
 162         err = set_etr3(pmcdev);
 163         if (err)
 164                 return err;
 165
 166         return len;
 167 }
 168 static DEVICE_ATTR_RW(etr3);
 169
 170 static struct attribute *pmc_attrs[] = {
 171         &dev_attr_etr3.attr,
 172         NULL
 173 };
 174
 175 static const struct attribute_group pmc_attr_group = {
 176         .attrs = pmc_attrs,
 177         .is_visible = etr3_is_visible,
 178 };
 179
 180 static const struct attribute_group *pmc_dev_groups[] = {
 181         &pmc_attr_group,
 182         NULL
 183 };
 184
 185 static int pmc_core_dev_state_get(void *data, u64 *val)
 186 {
 187         struct pmc *pmc = data;
 188         const struct pmc_reg_map *map = pmc->map;
 189         u32 value;
 190
 191         value = pmc_core_reg_read(pmc, map->slp_s0_offset);
 192         *val = pmc_core_adjust_slp_s0_step(pmc, value);
 193
 194         return 0;
 195 }
 196
 197 DEFINE_DEBUGFS_ATTRIBUTE(pmc_core_dev_state, pmc_core_dev_state_get, NULL, "%llu\n");
 198
 199 static int pmc_core_pson_residency_get(void *data, u64 *val)
 200 {
 201         struct pmc *pmc = data;
 202         const struct pmc_reg_map *map = pmc->map;
 203         u32 value;
 204
 205         value = pmc_core_reg_read(pmc, map->pson_residency_offset);
 206         *val = (u64)value * map->pson_residency_counter_step;
 207
 208         return 0;
 209 }
 210
 211 DEFINE_DEBUGFS_ATTRIBUTE(pmc_core_pson_residency, pmc_core_pson_residency_get, NULL, "%llu\n");
 212
 213 static int pmc_core_check_read_lock_bit(struct pmc *pmc)
 214 {
 215         u32 value;
 216
 217         value = pmc_core_reg_read(pmc, pmc->map->pm_cfg_offset);
 218         return value & BIT(pmc->map->pm_read_disable_bit);
 219 }
 220
 221 static void pmc_core_slps0_display(struct pmc *pmc, struct device *dev,
 222                                    struct seq_file *s)
 223 {
 224         const struct pmc_bit_map **maps = pmc->map->slps0_dbg_maps;
 225         const struct pmc_bit_map *map;
 226         int offset = pmc->map->slps0_dbg_offset;
 227         u32 data;
 228
 229         while (*maps) {
 230                 map = *maps;
 231                 data = pmc_core_reg_read(pmc, offset);
 232                 offset += 4;
 233                 while (map->name) {
 234                         if (dev)
 235                                 dev_info(dev, "SLP_S0_DBG: %-32s\tState: %s\n",
 236                                         map->name,
 237                                         data & map->bit_mask ? "Yes" : "No");
 238                         if (s)
 239                                 seq_printf(s, "SLP_S0_DBG: %-32s\tState: %s\n",
 240                                            map->name,
 241                                            data & map->bit_mask ? "Yes" : "No");
 242                         ++map;
 243                 }
 244                 ++maps;
 245         }
 246 }
 247
 248 static unsigned int pmc_core_lpm_get_arr_size(const struct pmc_bit_map **maps)
 249 {
 250         unsigned int idx;
 251
 252         for (idx = 0; maps[idx]; idx++)
 253                 ;/* Nothing */
 254
 255         return idx;
 256 }
 257
 258 static void pmc_core_lpm_display(struct pmc *pmc, struct device *dev,
 259                                  struct seq_file *s, u32 offset, int pmc_index,
 260                                  const char *str,
 261                                  const struct pmc_bit_map **maps)
 262 {
 263         unsigned int index, idx, len = 32, arr_size;
 264         u32 bit_mask, *lpm_regs;
 265
 266         arr_size = pmc_core_lpm_get_arr_size(maps);
 267         lpm_regs = kmalloc_array(arr_size, sizeof(*lpm_regs), GFP_KERNEL);
 268         if (!lpm_regs)
 269                 return;
 270
 271         for (index = 0; index < arr_size; index++) {
 272                 lpm_regs[index] = pmc_core_reg_read(pmc, offset);
 273                 offset += 4;
 274         }
 275
 276         for (idx = 0; idx < arr_size; idx++) {
 277                 if (dev)
 278                         dev_info(dev, "\nPMC%d:LPM_%s_%d:\t0x%x\n", pmc_index, str, idx,
 279                                 lpm_regs[idx]);
 280                 if (s)
 281                         seq_printf(s, "\nPMC%d:LPM_%s_%d:\t0x%x\n", pmc_index, str, idx,
 282                                    lpm_regs[idx]);
 283                 for (index = 0; maps[idx][index].name && index < len; index++) {
 284                         bit_mask = maps[idx][index].bit_mask;
 285                         if (dev)
 286                                 dev_info(dev, "PMC%d:%-30s %-30d\n", pmc_index,
 287                                         maps[idx][index].name,
 288                                         lpm_regs[idx] & bit_mask ? 1 : 0);
 289                         if (s)
 290                                 seq_printf(s, "PMC%d:%-30s %-30d\n", pmc_index,
 291                                            maps[idx][index].name,
 292                                            lpm_regs[idx] & bit_mask ? 1 : 0);
 293                 }
 294         }
 295
 296         kfree(lpm_regs);
 297 }
 298
 299 static bool slps0_dbg_latch;
 300
 301 static inline u8 pmc_core_reg_read_byte(struct pmc *pmc, int offset)
 302 {
 303         return readb(pmc->regbase + offset);
 304 }
 305
 306 static void pmc_core_display_map(struct seq_file *s, int index, int idx, int ip,
 307                                  int pmc_index, u8 pf_reg, const struct pmc_bit_map **pf_map)
 308 {
 309         seq_printf(s, "PMC%d:PCH IP: %-2d - %-32s\tState: %s\n",
 310                    pmc_index, ip, pf_map[idx][index].name,
 311                    pf_map[idx][index].bit_mask & pf_reg ? "Off" : "On");
 312 }
 313
 314 static int pmc_core_ppfear_show(struct seq_file *s, void *unused)
 315 {
 316         struct pmc_dev *pmcdev = s->private;
 317         unsigned int i;
 318
 319         for (i = 0; i < ARRAY_SIZE(pmcdev->pmcs); ++i) {
 320                 struct pmc *pmc = pmcdev->pmcs[i];
 321                 const struct pmc_bit_map **maps;
 322                 u8 pf_regs[PPFEAR_MAX_NUM_ENTRIES];
 323                 unsigned int index, iter, idx, ip = 0;
 324
 325                 if (!pmc)
 326                         continue;
 327
 328                 maps = pmc->map->pfear_sts;
 329                 iter = pmc->map->ppfear0_offset;
 330
 331                 for (index = 0; index < pmc->map->ppfear_buckets &&
 332                      index < PPFEAR_MAX_NUM_ENTRIES; index++, iter++)
 333                         pf_regs[index] = pmc_core_reg_read_byte(pmc, iter);
 334
 335                 for (idx = 0; maps[idx]; idx++) {
 336                         for (index = 0; maps[idx][index].name &&
 337                              index < pmc->map->ppfear_buckets * 8; ip++, index++)
 338                                 pmc_core_display_map(s, index, idx, ip, i,
 339                                                      pf_regs[index / 8], maps);
 340                 }
 341         }
 342
 343         return 0;
 344 }
 345 DEFINE_SHOW_ATTRIBUTE(pmc_core_ppfear);
 346
 347 /* This function should return link status, 0 means ready */
 348 static int pmc_core_mtpmc_link_status(struct pmc *pmc)
 349 {
 350         u32 value;
 351
 352         value = pmc_core_reg_read(pmc, SPT_PMC_PM_STS_OFFSET);
 353         return value & BIT(SPT_PMC_MSG_FULL_STS_BIT);
 354 }
 355
 356 static int pmc_core_send_msg(struct pmc *pmc, u32 *addr_xram)
 357 {
 358         u32 dest;
 359         int timeout;
 360
 361         for (timeout = NUM_RETRIES; timeout > 0; timeout--) {
 362                 if (pmc_core_mtpmc_link_status(pmc) == 0)
 363                         break;
 364                 msleep(5);
 365         }
 366
 367         if (timeout <= 0 && pmc_core_mtpmc_link_status(pmc))
 368                 return -EBUSY;
 369
 370         dest = (*addr_xram & MTPMC_MASK) | (1U << 1);
 371         pmc_core_reg_write(pmc, SPT_PMC_MTPMC_OFFSET, dest);
 372         return 0;
 373 }
 374
 375 static int pmc_core_mphy_pg_show(struct seq_file *s, void *unused)
 376 {
 377         struct pmc_dev *pmcdev = s->private;
 378         struct pmc *pmc = pmcdev->pmcs[PMC_IDX_MAIN];
 379         const struct pmc_bit_map *map = pmc->map->mphy_sts;
 380         u32 mphy_core_reg_low, mphy_core_reg_high;
 381         u32 val_low, val_high;
 382         unsigned int index;
 383         int err = 0;
 384
 385         if (pmcdev->pmc_xram_read_bit) {
 386                 seq_puts(s, "Access denied: please disable PMC_READ_DISABLE setting in BIOS.");
 387                 return 0;
 388         }
 389
 390         mphy_core_reg_low  = (SPT_PMC_MPHY_CORE_STS_0 << 16);
 391         mphy_core_reg_high = (SPT_PMC_MPHY_CORE_STS_1 << 16);
 392
 393         guard(mutex)(&pmcdev->lock);
 394
 395         err = pmc_core_send_msg(pmc, &mphy_core_reg_low);
 396         if (err)
 397                 return err;
 398
 399         msleep(10);
 400         val_low = pmc_core_reg_read(pmc, SPT_PMC_MFPMC_OFFSET);
 401
 402         err = pmc_core_send_msg(pmc, &mphy_core_reg_high);
 403         if (err)
 404                 return err;
 405
 406         msleep(10);
 407         val_high = pmc_core_reg_read(pmc, SPT_PMC_MFPMC_OFFSET);
 408
 409         for (index = 0; index < 8 && map[index].name; index++) {
 410                 seq_printf(s, "%-32s\tState: %s\n",
 411                            map[index].name,
 412                            map[index].bit_mask & val_low ? "Not power gated" :
 413                            "Power gated");
 414         }
 415
 416         for (index = 8; map[index].name; index++) {
 417                 seq_printf(s, "%-32s\tState: %s\n",
 418                            map[index].name,
 419                            map[index].bit_mask & val_high ? "Not power gated" :
 420                            "Power gated");
 421         }
 422
 423         return 0;
 424 }
 425 DEFINE_SHOW_ATTRIBUTE(pmc_core_mphy_pg);
 426
 427 static int pmc_core_pll_show(struct seq_file *s, void *unused)
 428 {
 429         struct pmc_dev *pmcdev = s->private;
 430         struct pmc *pmc = pmcdev->pmcs[PMC_IDX_MAIN];
 431         const struct pmc_bit_map *map = pmc->map->pll_sts;
 432         u32 mphy_common_reg, val;
 433         unsigned int index;
 434         int err = 0;
 435
 436         if (pmcdev->pmc_xram_read_bit) {
 437                 seq_puts(s, "Access denied: please disable PMC_READ_DISABLE setting in BIOS.");
 438                 return 0;
 439         }
 440
 441         mphy_common_reg  = (SPT_PMC_MPHY_COM_STS_0 << 16);
 442         guard(mutex)(&pmcdev->lock);
 443
 444         err = pmc_core_send_msg(pmc, &mphy_common_reg);
 445         if (err)
 446                 return err;
 447
 448         /* Observed PMC HW response latency for MTPMC-MFPMC is ~10 ms */
 449         msleep(10);
 450         val = pmc_core_reg_read(pmc, SPT_PMC_MFPMC_OFFSET);
 451
 452         for (index = 0; map[index].name ; index++) {
 453                 seq_printf(s, "%-32s\tState: %s\n",
 454                            map[index].name,
 455                            map[index].bit_mask & val ? "Active" : "Idle");
 456         }
 457
 458         return 0;
 459 }
 460 DEFINE_SHOW_ATTRIBUTE(pmc_core_pll);
 461
 462 int pmc_core_send_ltr_ignore(struct pmc_dev *pmcdev, u32 value, int ignore)
 463 {
 464         struct pmc *pmc;
 465         const struct pmc_reg_map *map;
 466         u32 reg;
 467         unsigned int pmc_index;
 468         int ltr_index;
 469
 470         ltr_index = value;
 471         /* For platforms with multiple pmcs, ltr index value given by user
 472          * is based on the contiguous indexes from ltr_show output.
 473          * pmc index and ltr index needs to be calculated from it.
 474          */
 475         for (pmc_index = 0; pmc_index < ARRAY_SIZE(pmcdev->pmcs) && ltr_index >= 0; pmc_index++) {
 476                 pmc = pmcdev->pmcs[pmc_index];
 477
 478                 if (!pmc)
 479                         continue;
 480
 481                 map = pmc->map;
 482                 if (ltr_index <= map->ltr_ignore_max)
 483                         break;
 484
 485                 /* Along with IP names, ltr_show map includes CURRENT_PLATFORM
 486                  * and AGGREGATED_SYSTEM values per PMC. Take these two index
 487                  * values into account in ltr_index calculation. Also, to start
 488                  * ltr index from zero for next pmc, subtract it by 1.
 489                  */
 490                 ltr_index = ltr_index - (map->ltr_ignore_max + 2) - 1;
 491         }
 492
 493         if (pmc_index >= ARRAY_SIZE(pmcdev->pmcs) || ltr_index < 0)
 494                 return -EINVAL;
 495
 496         pr_debug("ltr_ignore for pmc%d: ltr_index:%d\n", pmc_index, ltr_index);
 497
 498         guard(mutex)(&pmcdev->lock);
 499
 500         reg = pmc_core_reg_read(pmc, map->ltr_ignore_offset);
 501         if (ignore)
 502                 reg |= BIT(ltr_index);
 503         else
 504                 reg &= ~BIT(ltr_index);
 505         pmc_core_reg_write(pmc, map->ltr_ignore_offset, reg);
 506
 507         return 0;
 508 }
 509
 510 static ssize_t pmc_core_ltr_write(struct pmc_dev *pmcdev,
 511                                   const char __user *userbuf,
 512                                   size_t count, int ignore)
 513 {
 514         u32 value;
 515         int err;
 516
 517         err = kstrtou32_from_user(userbuf, count, 10, &value);
 518         if (err)
 519                 return err;
 520
 521         err = pmc_core_send_ltr_ignore(pmcdev, value, ignore);
 522
 523         return err ?: count;
 524 }
 525
 526 static ssize_t pmc_core_ltr_ignore_write(struct file *file,
 527                                          const char __user *userbuf,
 528                                          size_t count, loff_t *ppos)
 529 {
 530         struct seq_file *s = file->private_data;
 531         struct pmc_dev *pmcdev = s->private;
 532
 533         return pmc_core_ltr_write(pmcdev, userbuf, count, 1);
 534 }
 535
 536 static int pmc_core_ltr_ignore_show(struct seq_file *s, void *unused)
 537 {
 538         return 0;
 539 }
 540 DEFINE_SHOW_STORE_ATTRIBUTE(pmc_core_ltr_ignore);
 541
 542 static ssize_t pmc_core_ltr_restore_write(struct file *file,
 543                                           const char __user *userbuf,
 544                                           size_t count, loff_t *ppos)
 545 {
 546         struct seq_file *s = file->private_data;
 547         struct pmc_dev *pmcdev = s->private;
 548
 549         return pmc_core_ltr_write(pmcdev, userbuf, count, 0);
 550 }
 551
 552 static int pmc_core_ltr_restore_show(struct seq_file *s, void *unused)
 553 {
 554         return 0;
 555 }
 556 DEFINE_SHOW_STORE_ATTRIBUTE(pmc_core_ltr_restore);
 557
 558 static void pmc_core_slps0_dbg_latch(struct pmc_dev *pmcdev, bool reset)
 559 {
 560         struct pmc *pmc = pmcdev->pmcs[PMC_IDX_MAIN];
 561         const struct pmc_reg_map *map = pmc->map;
 562         u32 fd;
 563
 564         guard(mutex)(&pmcdev->lock);
 565
 566         if (!reset && !slps0_dbg_latch)
 567                 return;
 568
 569         fd = pmc_core_reg_read(pmc, map->slps0_dbg_offset);
 570         if (reset)
 571                 fd &= ~CNP_PMC_LATCH_SLPS0_EVENTS;
 572         else
 573                 fd |= CNP_PMC_LATCH_SLPS0_EVENTS;
 574         pmc_core_reg_write(pmc, map->slps0_dbg_offset, fd);
 575
 576         slps0_dbg_latch = false;
 577 }
 578
 579 static int pmc_core_slps0_dbg_show(struct seq_file *s, void *unused)
 580 {
 581         struct pmc_dev *pmcdev = s->private;
 582
 583         pmc_core_slps0_dbg_latch(pmcdev, false);
 584         pmc_core_slps0_display(pmcdev->pmcs[PMC_IDX_MAIN], NULL, s);
 585         pmc_core_slps0_dbg_latch(pmcdev, true);
 586
 587         return 0;
 588 }
 589 DEFINE_SHOW_ATTRIBUTE(pmc_core_slps0_dbg);
 590
 591 static u32 convert_ltr_scale(u32 val)
 592 {
 593         /*
 594          * As per PCIE specification supporting document
 595          * ECN_LatencyTolnReporting_14Aug08.pdf the Latency
 596          * Tolerance Reporting data payload is encoded in a
 597          * 3 bit scale and 10 bit value fields. Values are
 598          * multiplied by the indicated scale to yield an absolute time
 599          * value, expressible in a range from 1 nanosecond to
 600          * 2^25*(2^10-1) = 34,326,183,936 nanoseconds.
 601          *
 602          * scale encoding is as follows:
 603          *
 604          * ----------------------------------------------
 605          * |scale factor        |       Multiplier (ns) |
 606          * ----------------------------------------------
 607          * |    0               |       1               |
 608          * |    1               |       32              |
 609          * |    2               |       1024            |
 610          * |    3               |       32768           |
 611          * |    4               |       1048576         |
 612          * |    5               |       33554432        |
 613          * |    6               |       Invalid         |
 614          * |    7               |       Invalid         |
 615          * ----------------------------------------------
 616          */
 617         if (val > 5) {
 618                 pr_warn("Invalid LTR scale factor.\n");
 619                 return 0;
 620         }
 621
 622         return 1U << (5 * val);
 623 }
 624
 625 static int pmc_core_ltr_show(struct seq_file *s, void *unused)
 626 {
 627         struct pmc_dev *pmcdev = s->private;
 628         u64 decoded_snoop_ltr, decoded_non_snoop_ltr;
 629         u32 ltr_raw_data, scale, val;
 630         u16 snoop_ltr, nonsnoop_ltr;
 631         unsigned int i, index, ltr_index = 0;
 632
 633         for (i = 0; i < ARRAY_SIZE(pmcdev->pmcs); ++i) {
 634                 struct pmc *pmc;
 635                 const struct pmc_bit_map *map;
 636                 u32 ltr_ign_reg;
 637
 638                 pmc = pmcdev->pmcs[i];
 639                 if (!pmc)
 640                         continue;
 641
 642                 scoped_guard(mutex, &pmcdev->lock)
 643                         ltr_ign_reg = pmc_core_reg_read(pmc, pmc->map->ltr_ignore_offset);
 644
 645                 map = pmc->map->ltr_show_sts;
 646                 for (index = 0; map[index].name; index++) {
 647                         bool ltr_ign_data;
 648
 649                         if (index > pmc->map->ltr_ignore_max)
 650                                 ltr_ign_data = false;
 651                         else
 652                                 ltr_ign_data = ltr_ign_reg & BIT(index);
 653
 654                         decoded_snoop_ltr = decoded_non_snoop_ltr = 0;
 655                         ltr_raw_data = pmc_core_reg_read(pmc,
 656                                                          map[index].bit_mask);
 657                         snoop_ltr = ltr_raw_data & ~MTPMC_MASK;
 658                         nonsnoop_ltr = (ltr_raw_data >> 0x10) & ~MTPMC_MASK;
 659
 660                         if (FIELD_GET(LTR_REQ_NONSNOOP, ltr_raw_data)) {
 661                                 scale = FIELD_GET(LTR_DECODED_SCALE, nonsnoop_ltr);
 662                                 val = FIELD_GET(LTR_DECODED_VAL, nonsnoop_ltr);
 663                                 decoded_non_snoop_ltr = val * convert_ltr_scale(scale);
 664                         }
 665                         if (FIELD_GET(LTR_REQ_SNOOP, ltr_raw_data)) {
 666                                 scale = FIELD_GET(LTR_DECODED_SCALE, snoop_ltr);
 667                                 val = FIELD_GET(LTR_DECODED_VAL, snoop_ltr);
 668                                 decoded_snoop_ltr = val * convert_ltr_scale(scale);
 669                         }
 670
 671                         seq_printf(s, "%d\tPMC%d:%-32s\tLTR: RAW: 0x%-16x\tNon-Snoop(ns): %-16llu\tSnoop(ns): %-16llu\tLTR_IGNORE: %d\n",
 672                                    ltr_index, i, map[index].name, ltr_raw_data,
 673                                    decoded_non_snoop_ltr,
 674                                    decoded_snoop_ltr, ltr_ign_data);
 675                         ltr_index++;
 676                 }
 677         }
 678         return 0;
 679 }
 680 DEFINE_SHOW_ATTRIBUTE(pmc_core_ltr);
 681
 682 static int pmc_core_s0ix_blocker_show(struct seq_file *s, void *unused)
 683 {
 684         struct pmc_dev *pmcdev = s->private;
 685         unsigned int pmcidx;
 686
 687         for (pmcidx = 0; pmcidx < ARRAY_SIZE(pmcdev->pmcs); pmcidx++) {
 688                 const struct pmc_bit_map **maps;
 689                 unsigned int arr_size, r_idx;
 690                 u32 offset, counter;
 691                 struct pmc *pmc;
 692
 693                 pmc = pmcdev->pmcs[pmcidx];
 694                 if (!pmc)
 695                         continue;
 696                 maps = pmc->map->s0ix_blocker_maps;
 697                 offset = pmc->map->s0ix_blocker_offset;
 698                 arr_size = pmc_core_lpm_get_arr_size(maps);
 699
 700                 for (r_idx = 0; r_idx < arr_size; r_idx++) {
 701                         const struct pmc_bit_map *map;
 702
 703                         for (map = maps[r_idx]; map->name; map++) {
 704                                 if (!map->blk)
 705                                         continue;
 706                                 counter = pmc_core_reg_read(pmc, offset);
 707                                 seq_printf(s, "PMC%d:%-30s %-30d\n", pmcidx,
 708                                            map->name, counter);
 709                                 offset += map->blk * S0IX_BLK_SIZE;
 710                         }
 711                 }
 712         }
 713         return 0;
 714 }
 715 DEFINE_SHOW_ATTRIBUTE(pmc_core_s0ix_blocker);
 716
 717 static void pmc_core_ltr_ignore_all(struct pmc_dev *pmcdev)
 718 {
 719         unsigned int i;
 720
 721         for (i = 0; i < ARRAY_SIZE(pmcdev->pmcs); i++) {
 722                 struct pmc *pmc;
 723                 u32 ltr_ign;
 724
 725                 pmc = pmcdev->pmcs[i];
 726                 if (!pmc)
 727                         continue;
 728
 729                 guard(mutex)(&pmcdev->lock);
 730                 pmc->ltr_ign = pmc_core_reg_read(pmc, pmc->map->ltr_ignore_offset);
 731
 732                 /* ltr_ignore_max is the max index value for LTR ignore register */
 733                 ltr_ign = pmc->ltr_ign | GENMASK(pmc->map->ltr_ignore_max, 0);
 734                 pmc_core_reg_write(pmc, pmc->map->ltr_ignore_offset, ltr_ign);
 735         }
 736
 737         /*
 738          * Ignoring ME during suspend is blocking platforms with ADL PCH to get to
 739          * deeper S0ix substate.
 740          */
 741         pmc_core_send_ltr_ignore(pmcdev, 6, 0);
 742 }
 743
 744 static void pmc_core_ltr_restore_all(struct pmc_dev *pmcdev)
 745 {
 746         unsigned int i;
 747
 748         for (i = 0; i < ARRAY_SIZE(pmcdev->pmcs); i++) {
 749                 struct pmc *pmc;
 750
 751                 pmc = pmcdev->pmcs[i];
 752                 if (!pmc)
 753                         continue;
 754
 755                 guard(mutex)(&pmcdev->lock);
 756                 pmc_core_reg_write(pmc, pmc->map->ltr_ignore_offset, pmc->ltr_ign);
 757         }
 758 }
 759
 760 static inline u64 adjust_lpm_residency(struct pmc *pmc, u32 offset,
 761                                        const int lpm_adj_x2)
 762 {
 763         u64 lpm_res = pmc_core_reg_read(pmc, offset);
 764
 765         return GET_X2_COUNTER((u64)lpm_adj_x2 * lpm_res);
 766 }
 767
 768 static int pmc_core_substate_res_show(struct seq_file *s, void *unused)
 769 {
 770         struct pmc_dev *pmcdev = s->private;
 771         struct pmc *pmc = pmcdev->pmcs[PMC_IDX_MAIN];
 772         const int lpm_adj_x2 = pmc->map->lpm_res_counter_step_x2;
 773         u32 offset = pmc->map->lpm_residency_offset;
 774         int mode;
 775
 776         seq_printf(s, "%-10s %-15s\n", "Substate", "Residency");
 777
 778         pmc_for_each_mode(mode, pmcdev) {
 779                 seq_printf(s, "%-10s %-15llu\n", pmc_lpm_modes[mode],
 780                            adjust_lpm_residency(pmc, offset + (4 * mode), lpm_adj_x2));
 781         }
 782
 783         return 0;
 784 }
 785 DEFINE_SHOW_ATTRIBUTE(pmc_core_substate_res);
 786
 787 static int pmc_core_substate_sts_regs_show(struct seq_file *s, void *unused)
 788 {
 789         struct pmc_dev *pmcdev = s->private;
 790         unsigned int i;
 791
 792         for (i = 0; i < ARRAY_SIZE(pmcdev->pmcs); ++i) {
 793                 struct pmc *pmc = pmcdev->pmcs[i];
 794                 const struct pmc_bit_map **maps;
 795                 u32 offset;
 796
 797                 if (!pmc)
 798                         continue;
 799                 maps = pmc->map->lpm_sts;
 800                 offset = pmc->map->lpm_status_offset;
 801                 pmc_core_lpm_display(pmc, NULL, s, offset, i, "STATUS", maps);
 802         }
 803
 804         return 0;
 805 }
 806 DEFINE_SHOW_ATTRIBUTE(pmc_core_substate_sts_regs);
 807
 808 static int pmc_core_substate_l_sts_regs_show(struct seq_file *s, void *unused)
 809 {
 810         struct pmc_dev *pmcdev = s->private;
 811         unsigned int i;
 812
 813         for (i = 0; i < ARRAY_SIZE(pmcdev->pmcs); ++i) {
 814                 struct pmc *pmc = pmcdev->pmcs[i];
 815                 const struct pmc_bit_map **maps;
 816                 u32 offset;
 817
 818                 if (!pmc)
 819                         continue;
 820                 maps = pmc->map->lpm_sts;
 821                 offset = pmc->map->lpm_live_status_offset;
 822                 pmc_core_lpm_display(pmc, NULL, s, offset, i, "LIVE_STATUS", maps);
 823         }
 824
 825         return 0;
 826 }
 827 DEFINE_SHOW_ATTRIBUTE(pmc_core_substate_l_sts_regs);
 828
 829 static void pmc_core_substate_req_header_show(struct seq_file *s, int pmc_index)
 830 {
 831         struct pmc_dev *pmcdev = s->private;
 832         int mode;
 833
 834         seq_printf(s, "%30s |", "Element");
 835         pmc_for_each_mode(mode, pmcdev)
 836                 seq_printf(s, " %9s |", pmc_lpm_modes[mode]);
 837
 838         seq_printf(s, " %9s |", "Status");
 839         seq_printf(s, " %11s |\n", "Live Status");
 840 }
 841
 842 static int pmc_core_substate_req_regs_show(struct seq_file *s, void *unused)
 843 {
 844         struct pmc_dev *pmcdev = s->private;
 845         u32 sts_offset;
 846         u32 sts_offset_live;
 847         u32 *lpm_req_regs;
 848         unsigned int mp, pmc_index;
 849         int num_maps;
 850
 851         for (pmc_index = 0; pmc_index < ARRAY_SIZE(pmcdev->pmcs); ++pmc_index) {
 852                 struct pmc *pmc = pmcdev->pmcs[pmc_index];
 853                 const struct pmc_bit_map **maps;
 854
 855                 if (!pmc)
 856                         continue;
 857
 858                 maps = pmc->map->lpm_sts;
 859                 num_maps = pmc->map->lpm_num_maps;
 860                 sts_offset = pmc->map->lpm_status_offset;
 861                 sts_offset_live = pmc->map->lpm_live_status_offset;
 862                 lpm_req_regs = pmc->lpm_req_regs;
 863
 864                 /*
 865                  * When there are multiple PMCs, though the PMC may exist, the
 866                  * requirement register discovery could have failed so check
 867                  * before accessing.
 868                  */
 869                 if (!lpm_req_regs)
 870                         continue;
 871
 872                 /* Display the header */
 873                 pmc_core_substate_req_header_show(s, pmc_index);
 874
 875                 /* Loop over maps */
 876                 for (mp = 0; mp < num_maps; mp++) {
 877                         u32 req_mask = 0;
 878                         u32 lpm_status;
 879                         u32 lpm_status_live;
 880                         const struct pmc_bit_map *map;
 881                         int mode, i, len = 32;
 882
 883                         /*
 884                          * Capture the requirements and create a mask so that we only
 885                          * show an element if it's required for at least one of the
 886                          * enabled low power modes
 887                          */
 888                         pmc_for_each_mode(mode, pmcdev)
 889                                 req_mask |= lpm_req_regs[mp + (mode * num_maps)];
 890
 891                         /* Get the last latched status for this map */
 892                         lpm_status = pmc_core_reg_read(pmc, sts_offset + (mp * 4));
 893
 894                         /* Get the runtime status for this map */
 895                         lpm_status_live = pmc_core_reg_read(pmc, sts_offset_live + (mp * 4));
 896
 897                         /*  Loop over elements in this map */
 898                         map = maps[mp];
 899                         for (i = 0; map[i].name && i < len; i++) {
 900                                 u32 bit_mask = map[i].bit_mask;
 901
 902                                 if (!(bit_mask & req_mask)) {
 903                                         /*
 904                                          * Not required for any enabled states
 905                                          * so don't display
 906                                          */
 907                                         continue;
 908                                 }
 909
 910                                 /* Display the element name in the first column */
 911                                 seq_printf(s, "pmc%d: %26s |", pmc_index, map[i].name);
 912
 913                                 /* Loop over the enabled states and display if required */
 914                                 pmc_for_each_mode(mode, pmcdev) {
 915                                         bool required = lpm_req_regs[mp + (mode * num_maps)] &
 916                                                         bit_mask;
 917                                         seq_printf(s, " %9s |", required ? "Required" : " ");
 918                                 }
 919
 920                                 /* In Status column, show the last captured state of this agent */
 921                                 seq_printf(s, " %9s |", lpm_status & bit_mask ? "Yes" : " ");
 922
 923                                 /* In Live status column, show the live state of this agent */
 924                                 seq_printf(s, " %11s |", lpm_status_live & bit_mask ? "Yes" : " ");
 925
 926                                 seq_puts(s, "\n");
 927                         }
 928                 }
 929         }
 930         return 0;
 931 }
 932 DEFINE_SHOW_ATTRIBUTE(pmc_core_substate_req_regs);
 933
 934 static unsigned int pmc_core_get_crystal_freq(void)
 935 {
 936         unsigned int eax_denominator, ebx_numerator, ecx_hz, edx;
 937
 938         if (boot_cpu_data.cpuid_level < 0x15)
 939                 return 0;
 940
 941         eax_denominator = ebx_numerator = ecx_hz = edx = 0;
 942
 943         /* CPUID 15H TSC/Crystal ratio, plus optionally Crystal Hz */
 944         cpuid(0x15, &eax_denominator, &ebx_numerator, &ecx_hz, &edx);
 945
 946         if (ebx_numerator == 0 || eax_denominator == 0)
 947                 return 0;
 948
 949         return ecx_hz;
 950 }
 951
 952 static int pmc_core_die_c6_us_show(struct seq_file *s, void *unused)
 953 {
 954         struct pmc_dev *pmcdev = s->private;
 955         u64 die_c6_res, count;
 956         int ret;
 957
 958         if (!pmcdev->crystal_freq) {
 959                 dev_warn_once(&pmcdev->pdev->dev, "Crystal frequency unavailable\n");
 960                 return -ENXIO;
 961         }
 962
 963         ret = pmt_telem_read(pmcdev->punit_ep, pmcdev->die_c6_offset,
 964                              &count, 1);
 965         if (ret)
 966                 return ret;
 967
 968         die_c6_res = div64_u64(count * HZ_PER_MHZ, pmcdev->crystal_freq);
 969         seq_printf(s, "%llu\n", die_c6_res);
 970
 971         return 0;
 972 }
 973 DEFINE_SHOW_ATTRIBUTE(pmc_core_die_c6_us);
 974
 975 static int pmc_core_lpm_latch_mode_show(struct seq_file *s, void *unused)
 976 {
 977         struct pmc_dev *pmcdev = s->private;
 978         struct pmc *pmc = pmcdev->pmcs[PMC_IDX_MAIN];
 979         bool c10;
 980         u32 reg;
 981         int mode;
 982
 983         reg = pmc_core_reg_read(pmc, pmc->map->lpm_sts_latch_en_offset);
 984         if (reg & LPM_STS_LATCH_MODE) {
 985                 seq_puts(s, "c10");
 986                 c10 = false;
 987         } else {
 988                 seq_puts(s, "[c10]");
 989                 c10 = true;
 990         }
 991
 992         pmc_for_each_mode(mode, pmcdev) {
 993                 if ((BIT(mode) & reg) && !c10)
 994                         seq_printf(s, " [%s]", pmc_lpm_modes[mode]);
 995                 else
 996                         seq_printf(s, " %s", pmc_lpm_modes[mode]);
 997         }
 998
 999         seq_puts(s, " clear\n");
1000
1001         return 0;
1002 }
1003
1004 static ssize_t pmc_core_lpm_latch_mode_write(struct file *file,
1005                                              const char __user *userbuf,
1006                                              size_t count, loff_t *ppos)
1007 {
1008         struct seq_file *s = file->private_data;
1009         struct pmc_dev *pmcdev = s->private;
1010         struct pmc *pmc = pmcdev->pmcs[PMC_IDX_MAIN];
1011         bool clear = false, c10 = false;
1012         unsigned char buf[8];
1013         int m, mode;
1014         u32 reg;
1015
1016         if (count > sizeof(buf) - 1)
1017                 return -EINVAL;
1018         if (copy_from_user(buf, userbuf, count))
1019                 return -EFAULT;
1020         buf[count] = '\0';
1021
1022         /*
1023          * Allowed strings are:
1024          *      Any enabled substate, e.g. 'S0i2.0'
1025          *      'c10'
1026          *      'clear'
1027          */
1028         mode = sysfs_match_string(pmc_lpm_modes, buf);
1029
1030         /* Check string matches enabled mode */
1031         pmc_for_each_mode(m, pmcdev)
1032                 if (mode == m)
1033                         break;
1034
1035         if (mode != m || mode < 0) {
1036                 if (sysfs_streq(buf, "clear"))
1037                         clear = true;
1038                 else if (sysfs_streq(buf, "c10"))
1039                         c10 = true;
1040                 else
1041                         return -EINVAL;
1042         }
1043
1044         if (clear) {
1045                 guard(mutex)(&pmcdev->lock);
1046
1047                 reg = pmc_core_reg_read(pmc, pmc->map->etr3_offset);
1048                 reg |= ETR3_CLEAR_LPM_EVENTS;
1049                 pmc_core_reg_write(pmc, pmc->map->etr3_offset, reg);
1050
1051                 return count;
1052         }
1053
1054         if (c10) {
1055                 guard(mutex)(&pmcdev->lock);
1056
1057                 reg = pmc_core_reg_read(pmc, pmc->map->lpm_sts_latch_en_offset);
1058                 reg &= ~LPM_STS_LATCH_MODE;
1059                 pmc_core_reg_write(pmc, pmc->map->lpm_sts_latch_en_offset, reg);
1060
1061                 return count;
1062         }
1063
1064         /*
1065          * For LPM mode latching we set the latch enable bit and selected mode
1066          * and clear everything else.
1067          */
1068         reg = LPM_STS_LATCH_MODE | BIT(mode);
1069         guard(mutex)(&pmcdev->lock);
1070         pmc_core_reg_write(pmc, pmc->map->lpm_sts_latch_en_offset, reg);
1071
1072         return count;
1073 }
1074 DEFINE_PMC_CORE_ATTR_WRITE(pmc_core_lpm_latch_mode);
1075
1076 static int pmc_core_pkgc_show(struct seq_file *s, void *unused)
1077 {
1078         struct pmc *pmc = s->private;
1079         const struct pmc_bit_map *map = pmc->map->msr_sts;
1080         u64 pcstate_count;
1081         unsigned int index;
1082
1083         for (index = 0; map[index].name ; index++) {
1084                 if (rdmsrl_safe(map[index].bit_mask, &pcstate_count))
1085                         continue;
1086
1087                 pcstate_count *= 1000;
1088                 do_div(pcstate_count, tsc_khz);
1089                 seq_printf(s, "%-8s : %llu\n", map[index].name,
1090                            pcstate_count);
1091         }
1092
1093         return 0;
1094 }
1095 DEFINE_SHOW_ATTRIBUTE(pmc_core_pkgc);
1096
1097 static bool pmc_core_pri_verify(u32 lpm_pri, u8 *mode_order)
1098 {
1099         unsigned int i, j;
1100
1101         if (!lpm_pri)
1102                 return false;
1103         /*
1104          * Each byte contains the priority level for 2 modes (7:4 and 3:0).
1105          * In a 32 bit register this allows for describing 8 modes. Store the
1106          * levels and look for values out of range.
1107          */
1108         for (i = 0; i < 8; i++) {
1109                 int level = lpm_pri & GENMASK(3, 0);
1110
1111                 if (level >= LPM_MAX_NUM_MODES)
1112                         return false;
1113
1114                 mode_order[i] = level;
1115                 lpm_pri >>= 4;
1116         }
1117
1118         /* Check that we have unique values */
1119         for (i = 0; i < LPM_MAX_NUM_MODES - 1; i++)
1120                 for (j = i + 1; j < LPM_MAX_NUM_MODES; j++)
1121                         if (mode_order[i] == mode_order[j])
1122                                 return false;
1123
1124         return true;
1125 }
1126
1127 void pmc_core_get_low_power_modes(struct pmc_dev *pmcdev)
1128 {
1129         struct pmc *pmc = pmcdev->pmcs[PMC_IDX_MAIN];
1130         u8 pri_order[LPM_MAX_NUM_MODES] = LPM_DEFAULT_PRI;
1131         u8 mode_order[LPM_MAX_NUM_MODES];
1132         u32 lpm_pri;
1133         u32 lpm_en;
1134         unsigned int i;
1135         int mode, p;
1136
1137         /* Use LPM Maps to indicate support for substates */
1138         if (!pmc->map->lpm_num_maps)
1139                 return;
1140
1141         lpm_en = pmc_core_reg_read(pmc, pmc->map->lpm_en_offset);
1142         /* For MTL, BIT 31 is not an lpm mode but a enable bit.
1143          * Lower byte is enough to cover the number of lpm modes for all
1144          * platforms and hence mask the upper 3 bytes.
1145          */
1146         pmcdev->num_lpm_modes = hweight32(lpm_en & 0xFF);
1147
1148         /* Read 32 bit LPM_PRI register */
1149         lpm_pri = pmc_core_reg_read(pmc, pmc->map->lpm_priority_offset);
1150
1151
1152         /*
1153          * If lpm_pri value passes verification, then override the default
1154          * modes here. Otherwise stick with the default.
1155          */
1156         if (pmc_core_pri_verify(lpm_pri, mode_order))
1157                 /* Get list of modes in priority order */
1158                 for (mode = 0; mode < LPM_MAX_NUM_MODES; mode++)
1159                         pri_order[mode_order[mode]] = mode;
1160         else
1161                 dev_warn(&pmcdev->pdev->dev,
1162                          "Assuming a default substate order for this platform\n");
1163
1164         /*
1165          * Loop through all modes from lowest to highest priority,
1166          * and capture all enabled modes in order
1167          */
1168         i = 0;
1169         for (p = LPM_MAX_NUM_MODES - 1; p >= 0; p--) {
1170                 int mode = pri_order[p];
1171
1172                 if (!(BIT(mode) & lpm_en))
1173                         continue;
1174
1175                 pmcdev->lpm_en_modes[i++] = mode;
1176         }
1177 }
1178
1179 int get_primary_reg_base(struct pmc *pmc)
1180 {
1181         u64 slp_s0_addr;
1182
1183         if (lpit_read_residency_count_address(&slp_s0_addr)) {
1184                 pmc->base_addr = PMC_BASE_ADDR_DEFAULT;
1185
1186                 if (page_is_ram(PHYS_PFN(pmc->base_addr)))
1187                         return -ENODEV;
1188         } else {
1189                 pmc->base_addr = slp_s0_addr - pmc->map->slp_s0_offset;
1190         }
1191
1192         pmc->regbase = ioremap(pmc->base_addr, pmc->map->regmap_length);
1193         if (!pmc->regbase)
1194                 return -ENOMEM;
1195         return 0;
1196 }
1197
1198 void pmc_core_punit_pmt_init(struct pmc_dev *pmcdev, u32 guid)
1199 {
1200         struct telem_endpoint *ep;
1201         struct pci_dev *pcidev;
1202
1203         pcidev = pci_get_domain_bus_and_slot(0, 0, PCI_DEVFN(10, 0));
1204         if (!pcidev) {
1205                 dev_err(&pmcdev->pdev->dev, "PUNIT PMT device not found.");
1206                 return;
1207         }
1208
1209         ep = pmt_telem_find_and_register_endpoint(pcidev, guid, 0);
1210         pci_dev_put(pcidev);
1211         if (IS_ERR(ep)) {
1212                 dev_err(&pmcdev->pdev->dev,
1213                         "pmc_core: couldn't get DMU telem endpoint %ld",
1214                         PTR_ERR(ep));
1215                 return;
1216         }
1217
1218         pmcdev->punit_ep = ep;
1219
1220         pmcdev->has_die_c6 = true;
1221         pmcdev->die_c6_offset = MTL_PMT_DMU_DIE_C6_OFFSET;
1222 }
1223
1224 void pmc_core_set_device_d3(unsigned int device)
1225 {
1226         struct pci_dev *pcidev;
1227
1228         pcidev = pci_get_device(PCI_VENDOR_ID_INTEL, device, NULL);
1229         if (pcidev) {
1230                 if (!device_trylock(&pcidev->dev)) {
1231                         pci_dev_put(pcidev);
1232                         return;
1233                 }
1234                 if (!pcidev->dev.driver) {
1235                         dev_info(&pcidev->dev, "Setting to D3hot\n");
1236                         pci_set_power_state(pcidev, PCI_D3hot);
1237                 }
1238                 device_unlock(&pcidev->dev);
1239                 pci_dev_put(pcidev);
1240         }
1241 }
1242
1243 static bool pmc_core_is_pson_residency_enabled(struct pmc_dev *pmcdev)
1244 {
1245         struct platform_device *pdev = pmcdev->pdev;
1246         struct acpi_device *adev = ACPI_COMPANION(&pdev->dev);
1247         u8 val;
1248
1249         if (!adev)
1250                 return false;
1251
1252         if (fwnode_property_read_u8(acpi_fwnode_handle(adev),
1253                                     "intel-cec-pson-switching-enabled-in-s0",
1254                                     &val))
1255                 return false;
1256
1257         return val == 1;
1258 }
1259
1260 static void pmc_core_dbgfs_unregister(struct pmc_dev *pmcdev)
1261 {
1262         debugfs_remove_recursive(pmcdev->dbgfs_dir);
1263 }
1264
1265 static void pmc_core_dbgfs_register(struct pmc_dev *pmcdev)
1266 {
1267         struct pmc *primary_pmc = pmcdev->pmcs[PMC_IDX_MAIN];
1268         struct dentry *dir;
1269
1270         dir = debugfs_create_dir("pmc_core", NULL);
1271         pmcdev->dbgfs_dir = dir;
1272
1273         debugfs_create_file("slp_s0_residency_usec", 0444, dir, primary_pmc,
1274                             &pmc_core_dev_state);
1275
1276         if (primary_pmc->map->pfear_sts)
1277                 debugfs_create_file("pch_ip_power_gating_status", 0444, dir,
1278                                     pmcdev, &pmc_core_ppfear_fops);
1279
1280         debugfs_create_file("ltr_ignore", 0644, dir, pmcdev,
1281                             &pmc_core_ltr_ignore_fops);
1282
1283         debugfs_create_file("ltr_restore", 0200, dir, pmcdev, &pmc_core_ltr_restore_fops);
1284
1285         debugfs_create_file("ltr_show", 0444, dir, pmcdev, &pmc_core_ltr_fops);
1286
1287         if (primary_pmc->map->s0ix_blocker_maps)
1288                 debugfs_create_file("s0ix_blocker", 0444, dir, pmcdev, &pmc_core_s0ix_blocker_fops);
1289
1290         debugfs_create_file("package_cstate_show", 0444, dir, primary_pmc,
1291                             &pmc_core_pkgc_fops);
1292
1293         if (primary_pmc->map->pll_sts)
1294                 debugfs_create_file("pll_status", 0444, dir, pmcdev,
1295                                     &pmc_core_pll_fops);
1296
1297         if (primary_pmc->map->mphy_sts)
1298                 debugfs_create_file("mphy_core_lanes_power_gating_status",
1299                                     0444, dir, pmcdev,
1300                                     &pmc_core_mphy_pg_fops);
1301
1302         if (primary_pmc->map->slps0_dbg_maps) {
1303                 debugfs_create_file("slp_s0_debug_status", 0444,
1304                                     dir, pmcdev,
1305                                     &pmc_core_slps0_dbg_fops);
1306
1307                 debugfs_create_bool("slp_s0_dbg_latch", 0644,
1308                                     dir, &slps0_dbg_latch);
1309         }
1310
1311         if (primary_pmc->map->lpm_en_offset) {
1312                 debugfs_create_file("substate_residencies", 0444,
1313                                     pmcdev->dbgfs_dir, pmcdev,
1314                                     &pmc_core_substate_res_fops);
1315         }
1316
1317         if (primary_pmc->map->lpm_status_offset) {
1318                 debugfs_create_file("substate_status_registers", 0444,
1319                                     pmcdev->dbgfs_dir, pmcdev,
1320                                     &pmc_core_substate_sts_regs_fops);
1321                 debugfs_create_file("substate_live_status_registers", 0444,
1322                                     pmcdev->dbgfs_dir, pmcdev,
1323                                     &pmc_core_substate_l_sts_regs_fops);
1324                 debugfs_create_file("lpm_latch_mode", 0644,
1325                                     pmcdev->dbgfs_dir, pmcdev,
1326                                     &pmc_core_lpm_latch_mode_fops);
1327         }
1328
1329         if (primary_pmc->lpm_req_regs) {
1330                 debugfs_create_file("substate_requirements", 0444,
1331                                     pmcdev->dbgfs_dir, pmcdev,
1332                                     &pmc_core_substate_req_regs_fops);
1333         }
1334
1335         if (primary_pmc->map->pson_residency_offset && pmc_core_is_pson_residency_enabled(pmcdev)) {
1336                 debugfs_create_file("pson_residency_usec", 0444,
1337                                     pmcdev->dbgfs_dir, primary_pmc, &pmc_core_pson_residency);
1338         }
1339
1340         if (pmcdev->has_die_c6) {
1341                 debugfs_create_file("die_c6_us_show", 0444,
1342                                     pmcdev->dbgfs_dir, pmcdev,
1343                                     &pmc_core_die_c6_us_fops);
1344         }
1345 }
1346
1347 static const struct x86_cpu_id intel_pmc_core_ids[] = {
1348         X86_MATCH_VFM(INTEL_SKYLAKE_L,          spt_core_init),
1349         X86_MATCH_VFM(INTEL_SKYLAKE,            spt_core_init),
1350         X86_MATCH_VFM(INTEL_KABYLAKE_L,         spt_core_init),
1351         X86_MATCH_VFM(INTEL_KABYLAKE,           spt_core_init),
1352         X86_MATCH_VFM(INTEL_CANNONLAKE_L,       cnp_core_init),
1353         X86_MATCH_VFM(INTEL_ICELAKE_L,          icl_core_init),
1354         X86_MATCH_VFM(INTEL_ICELAKE_NNPI,       icl_core_init),
1355         X86_MATCH_VFM(INTEL_COMETLAKE,          cnp_core_init),
1356         X86_MATCH_VFM(INTEL_COMETLAKE_L,        cnp_core_init),
1357         X86_MATCH_VFM(INTEL_TIGERLAKE_L,        tgl_l_core_init),
1358         X86_MATCH_VFM(INTEL_TIGERLAKE,          tgl_core_init),
1359         X86_MATCH_VFM(INTEL_ATOM_TREMONT,       tgl_l_core_init),
1360         X86_MATCH_VFM(INTEL_ATOM_TREMONT_L,     icl_core_init),
1361         X86_MATCH_VFM(INTEL_ROCKETLAKE,         tgl_core_init),
1362         X86_MATCH_VFM(INTEL_ALDERLAKE_L,        tgl_l_core_init),
1363         X86_MATCH_VFM(INTEL_ATOM_GRACEMONT,     tgl_l_core_init),
1364         X86_MATCH_VFM(INTEL_ALDERLAKE,          adl_core_init),
1365         X86_MATCH_VFM(INTEL_RAPTORLAKE_P,       tgl_l_core_init),
1366         X86_MATCH_VFM(INTEL_RAPTORLAKE,         adl_core_init),
1367         X86_MATCH_VFM(INTEL_RAPTORLAKE_S,       adl_core_init),
1368         X86_MATCH_VFM(INTEL_METEORLAKE_L,       mtl_core_init),
1369         X86_MATCH_VFM(INTEL_ARROWLAKE,          arl_core_init),
1370         X86_MATCH_VFM(INTEL_LUNARLAKE_M,        lnl_core_init),
1371         {}
1372 };
1373
1374 MODULE_DEVICE_TABLE(x86cpu, intel_pmc_core_ids);
1375
1376 static const struct pci_device_id pmc_pci_ids[] = {
1377         { PCI_VDEVICE(INTEL, SPT_PMC_PCI_DEVICE_ID) },
1378         { }
1379 };
1380
1381 /*
1382  * This quirk can be used on those platforms where
1383  * the platform BIOS enforces 24Mhz crystal to shutdown
1384  * before PMC can assert SLP_S0#.
1385  */
1386 static bool xtal_ignore;
1387 static int quirk_xtal_ignore(const struct dmi_system_id *id)
1388 {
1389         xtal_ignore = true;
1390         return 0;
1391 }
1392
1393 static void pmc_core_xtal_ignore(struct pmc *pmc)
1394 {
1395         u32 value;
1396
1397         value = pmc_core_reg_read(pmc, pmc->map->pm_vric1_offset);
1398         /* 24MHz Crystal Shutdown Qualification Disable */
1399         value |= SPT_PMC_VRIC1_XTALSDQDIS;
1400         /* Low Voltage Mode Enable */
1401         value &= ~SPT_PMC_VRIC1_SLPS0LVEN;
1402         pmc_core_reg_write(pmc, pmc->map->pm_vric1_offset, value);
1403 }
1404
1405 static const struct dmi_system_id pmc_core_dmi_table[]  = {
1406         {
1407         .callback = quirk_xtal_ignore,
1408         .ident = "HP Elite x2 1013 G3",
1409         .matches = {
1410                 DMI_MATCH(DMI_SYS_VENDOR, "HP"),
1411                 DMI_MATCH(DMI_PRODUCT_NAME, "HP Elite x2 1013 G3"),
1412                 },
1413         },
1414         {}
1415 };
1416
1417 static void pmc_core_do_dmi_quirks(struct pmc *pmc)
1418 {
1419         dmi_check_system(pmc_core_dmi_table);
1420
1421         if (xtal_ignore)
1422                 pmc_core_xtal_ignore(pmc);
1423 }
1424
1425 static void pmc_core_clean_structure(struct platform_device *pdev)
1426 {
1427         struct pmc_dev *pmcdev = platform_get_drvdata(pdev);
1428         unsigned int i;
1429
1430         for (i = 0; i < ARRAY_SIZE(pmcdev->pmcs); ++i) {
1431                 struct pmc *pmc = pmcdev->pmcs[i];
1432
1433                 if (pmc)
1434                         iounmap(pmc->regbase);
1435         }
1436
1437         if (pmcdev->ssram_pcidev) {
1438                 pci_dev_put(pmcdev->ssram_pcidev);
1439                 pci_disable_device(pmcdev->ssram_pcidev);
1440         }
1441
1442         if (pmcdev->punit_ep)
1443                 pmt_telem_unregister_endpoint(pmcdev->punit_ep);
1444
1445         platform_set_drvdata(pdev, NULL);
1446         mutex_destroy(&pmcdev->lock);
1447 }
1448
1449 static int pmc_core_probe(struct platform_device *pdev)
1450 {
1451         static bool device_initialized;
1452         struct pmc_dev *pmcdev;
1453         const struct x86_cpu_id *cpu_id;
1454         int (*core_init)(struct pmc_dev *pmcdev);
1455         struct pmc *primary_pmc;
1456         int ret;
1457
1458         if (device_initialized)
1459                 return -ENODEV;
1460
1461         pmcdev = devm_kzalloc(&pdev->dev, sizeof(*pmcdev), GFP_KERNEL);
1462         if (!pmcdev)
1463                 return -ENOMEM;
1464
1465         pmcdev->crystal_freq = pmc_core_get_crystal_freq();
1466
1467         platform_set_drvdata(pdev, pmcdev);
1468         pmcdev->pdev = pdev;
1469
1470         cpu_id = x86_match_cpu(intel_pmc_core_ids);
1471         if (!cpu_id)
1472                 return -ENODEV;
1473
1474         core_init = (int (*)(struct pmc_dev *))cpu_id->driver_data;
1475
1476         /* Primary PMC */
1477         primary_pmc = devm_kzalloc(&pdev->dev, sizeof(*primary_pmc), GFP_KERNEL);
1478         if (!primary_pmc)
1479                 return -ENOMEM;
1480         pmcdev->pmcs[PMC_IDX_MAIN] = primary_pmc;
1481
1482         /* The last element in msr_map is empty */
1483         pmcdev->num_of_pkgc = ARRAY_SIZE(msr_map) - 1;
1484         pmcdev->pkgc_res_cnt = devm_kcalloc(&pdev->dev,
1485                                             pmcdev->num_of_pkgc,
1486                                             sizeof(*pmcdev->pkgc_res_cnt),
1487                                             GFP_KERNEL);
1488         if (!pmcdev->pkgc_res_cnt)
1489                 return -ENOMEM;
1490
1491         /*
1492          * Coffee Lake has CPU ID of Kaby Lake and Cannon Lake PCH. So here
1493          * Sunrisepoint PCH regmap can't be used. Use Cannon Lake PCH regmap
1494          * in this case.
1495          */
1496         if (core_init == spt_core_init && !pci_dev_present(pmc_pci_ids))
1497                 core_init = cnp_core_init;
1498
1499         mutex_init(&pmcdev->lock);
1500         ret = core_init(pmcdev);
1501         if (ret) {
1502                 pmc_core_clean_structure(pdev);
1503                 return ret;
1504         }
1505
1506         pmcdev->pmc_xram_read_bit = pmc_core_check_read_lock_bit(primary_pmc);
1507         pmc_core_do_dmi_quirks(primary_pmc);
1508
1509         pmc_core_dbgfs_register(pmcdev);
1510         pm_report_max_hw_sleep(FIELD_MAX(SLP_S0_RES_COUNTER_MASK) *
1511                                pmc_core_adjust_slp_s0_step(primary_pmc, 1));
1512
1513         device_initialized = true;
1514         dev_info(&pdev->dev, " initialized\n");
1515
1516         return 0;
1517 }
1518
1519 static void pmc_core_remove(struct platform_device *pdev)
1520 {
1521         struct pmc_dev *pmcdev = platform_get_drvdata(pdev);
1522         pmc_core_dbgfs_unregister(pmcdev);
1523         pmc_core_clean_structure(pdev);
1524 }
1525
1526 static bool warn_on_s0ix_failures;
1527 module_param(warn_on_s0ix_failures, bool, 0644);
1528 MODULE_PARM_DESC(warn_on_s0ix_failures, "Check and warn for S0ix failures");
1529
1530 static bool ltr_ignore_all_suspend = true;
1531 module_param(ltr_ignore_all_suspend, bool, 0644);
1532 MODULE_PARM_DESC(ltr_ignore_all_suspend, "Ignore all LTRs during suspend");
1533
1534 static __maybe_unused int pmc_core_suspend(struct device *dev)
1535 {
1536         struct pmc_dev *pmcdev = dev_get_drvdata(dev);
1537         struct pmc *pmc = pmcdev->pmcs[PMC_IDX_MAIN];
1538         unsigned int i;
1539
1540         if (pmcdev->suspend)
1541                 pmcdev->suspend(pmcdev);
1542
1543         if (ltr_ignore_all_suspend)
1544                 pmc_core_ltr_ignore_all(pmcdev);
1545
1546         /* Check if the syspend will actually use S0ix */
1547         if (pm_suspend_via_firmware())
1548                 return 0;
1549
1550         /* Save PKGC residency for checking later */
1551         for (i = 0; i < pmcdev->num_of_pkgc; i++) {
1552                 if (rdmsrl_safe(msr_map[i].bit_mask, &pmcdev->pkgc_res_cnt[i]))
1553                         return -EIO;
1554         }
1555
1556         /* Save S0ix residency for checking later */
1557         if (pmc_core_dev_state_get(pmc, &pmcdev->s0ix_counter))
1558                 return -EIO;
1559
1560         return 0;
1561 }
1562
1563 static inline bool pmc_core_is_deepest_pkgc_failed(struct pmc_dev *pmcdev)
1564 {
1565         u32 deepest_pkgc_msr = msr_map[pmcdev->num_of_pkgc - 1].bit_mask;
1566         u64 deepest_pkgc_residency;
1567
1568         if (rdmsrl_safe(deepest_pkgc_msr, &deepest_pkgc_residency))
1569                 return false;
1570
1571         if (deepest_pkgc_residency == pmcdev->pkgc_res_cnt[pmcdev->num_of_pkgc - 1])
1572                 return true;
1573
1574         return false;
1575 }
1576
1577 static inline bool pmc_core_is_s0ix_failed(struct pmc_dev *pmcdev)
1578 {
1579         u64 s0ix_counter;
1580
1581         if (pmc_core_dev_state_get(pmcdev->pmcs[PMC_IDX_MAIN], &s0ix_counter))
1582                 return false;
1583
1584         pm_report_hw_sleep_time((u32)(s0ix_counter - pmcdev->s0ix_counter));
1585
1586         if (s0ix_counter == pmcdev->s0ix_counter)
1587                 return true;
1588
1589         return false;
1590 }
1591
1592 int pmc_core_resume_common(struct pmc_dev *pmcdev)
1593 {
1594         struct device *dev = &pmcdev->pdev->dev;
1595         struct pmc *pmc = pmcdev->pmcs[PMC_IDX_MAIN];
1596         const struct pmc_bit_map **maps = pmc->map->lpm_sts;
1597         int offset = pmc->map->lpm_status_offset;
1598         unsigned int i;
1599
1600         /* Check if the syspend used S0ix */
1601         if (pm_suspend_via_firmware())
1602                 return 0;
1603
1604         if (!pmc_core_is_s0ix_failed(pmcdev))
1605                 return 0;
1606
1607         if (!warn_on_s0ix_failures)
1608                 return 0;
1609
1610         if (pmc_core_is_deepest_pkgc_failed(pmcdev)) {
1611                 /* S0ix failed because of deepest PKGC entry failure */
1612                 dev_info(dev, "CPU did not enter %s!!! (%s cnt=0x%llx)\n",
1613                          msr_map[pmcdev->num_of_pkgc - 1].name,
1614                          msr_map[pmcdev->num_of_pkgc - 1].name,
1615                          pmcdev->pkgc_res_cnt[pmcdev->num_of_pkgc - 1]);
1616
1617                 for (i = 0; i < pmcdev->num_of_pkgc; i++) {
1618                         u64 pc_cnt;
1619
1620                         if (!rdmsrl_safe(msr_map[i].bit_mask, &pc_cnt)) {
1621                                 dev_info(dev, "Prev %s cnt = 0x%llx, Current %s cnt = 0x%llx\n",
1622                                          msr_map[i].name, pmcdev->pkgc_res_cnt[i],
1623                                          msr_map[i].name, pc_cnt);
1624                         }
1625                 }
1626                 return 0;
1627         }
1628
1629         /* The real interesting case - S0ix failed - lets ask PMC why. */
1630         dev_warn(dev, "CPU did not enter SLP_S0!!! (S0ix cnt=%llu)\n",
1631                  pmcdev->s0ix_counter);
1632
1633         if (pmc->map->slps0_dbg_maps)
1634                 pmc_core_slps0_display(pmc, dev, NULL);
1635
1636         for (i = 0; i < ARRAY_SIZE(pmcdev->pmcs); ++i) {
1637                 struct pmc *pmc = pmcdev->pmcs[i];
1638
1639                 if (!pmc)
1640                         continue;
1641                 if (pmc->map->lpm_sts)
1642                         pmc_core_lpm_display(pmc, dev, NULL, offset, i, "STATUS", maps);
1643         }
1644
1645         return 0;
1646 }
1647
1648 static __maybe_unused int pmc_core_resume(struct device *dev)
1649 {
1650         struct pmc_dev *pmcdev = dev_get_drvdata(dev);
1651
1652         if (ltr_ignore_all_suspend)
1653                 pmc_core_ltr_restore_all(pmcdev);
1654
1655         if (pmcdev->resume)
1656                 return pmcdev->resume(pmcdev);
1657
1658         return pmc_core_resume_common(pmcdev);
1659 }
1660
1661 static const struct dev_pm_ops pmc_core_pm_ops = {
1662         SET_LATE_SYSTEM_SLEEP_PM_OPS(pmc_core_suspend, pmc_core_resume)
1663 };
1664
1665 static const struct acpi_device_id pmc_core_acpi_ids[] = {
1666         {"INT33A1", 0}, /* _HID for Intel Power Engine, _CID PNP0D80*/
1667         { }
1668 };
1669 MODULE_DEVICE_TABLE(acpi, pmc_core_acpi_ids);
1670
1671 static struct platform_driver pmc_core_driver = {
1672         .driver = {
1673                 .name = "intel_pmc_core",
1674                 .acpi_match_table = ACPI_PTR(pmc_core_acpi_ids),
1675                 .pm = &pmc_core_pm_ops,
1676                 .dev_groups = pmc_dev_groups,
1677         },
1678         .probe = pmc_core_probe,
1679         .remove = pmc_core_remove,
1680 };
1681
1682 module_platform_driver(pmc_core_driver);
1683
1684 MODULE_LICENSE("GPL v2");
1685 MODULE_DESCRIPTION("Intel PMC Core Driver");