[linux.git] / drivers / cpuidle / cpuidle-powernv.c

// SPDX-License-Identifier: GPL-2.0
/*
 *  cpuidle-powernv - idle state cpuidle driver.
 *  Adapted from drivers/cpuidle/cpuidle-pseries
 *
 */

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/moduleparam.h>
#include <linux/cpuidle.h>
#include <linux/cpu.h>
#include <linux/notifier.h>
#include <linux/clockchips.h>
#include <linux/of.h>
#include <linux/slab.h>

#include <asm/machdep.h>
#include <asm/firmware.h>
#include <asm/opal.h>
#include <asm/runlatch.h>
#include <asm/cpuidle.h>

/*
 * Expose only those Hardware idle states via the cpuidle framework
 * that have latency value below POWERNV_THRESHOLD_LATENCY_NS.
 */
#define POWERNV_THRESHOLD_LATENCY_NS 200000

static struct cpuidle_driver powernv_idle_driver = {
	.name             = "powernv_idle",
	.owner            = THIS_MODULE,
};

static int max_idle_state __read_mostly;
static struct cpuidle_state *cpuidle_state_table __read_mostly;

struct stop_psscr_table {
	u64 val;
	u64 mask;
};

static struct stop_psscr_table stop_psscr_table[CPUIDLE_STATE_MAX] __read_mostly;

static u64 snooze_timeout __read_mostly;
static bool snooze_timeout_en __read_mostly;

static int snooze_loop(struct cpuidle_device *dev,
			struct cpuidle_driver *drv,
			int index)
{
	u64 snooze_exit_time;

	set_thread_flag(TIF_POLLING_NRFLAG);

	local_irq_enable();

	snooze_exit_time = get_tb() + snooze_timeout;
	ppc64_runlatch_off();
	HMT_very_low();
	while (!need_resched()) {
		if (likely(snooze_timeout_en) && get_tb() > snooze_exit_time) {
			/*
			 * Task has not woken up but we are exiting the polling
			 * loop anyway. Require a barrier after polling is
			 * cleared to order subsequent test of need_resched().
			 */
			clear_thread_flag(TIF_POLLING_NRFLAG);
			smp_mb();
			break;
		}
	}

	HMT_medium();
	ppc64_runlatch_on();
	clear_thread_flag(TIF_POLLING_NRFLAG);

	local_irq_disable();

	return index;
}

static int nap_loop(struct cpuidle_device *dev,
			struct cpuidle_driver *drv,
			int index)
{
	power7_idle_type(PNV_THREAD_NAP);

	return index;
}

/* Register for fastsleep only in oneshot mode of broadcast */
#ifdef CONFIG_TICK_ONESHOT
static int fastsleep_loop(struct cpuidle_device *dev,
				struct cpuidle_driver *drv,
				int index)
{
	unsigned long old_lpcr = mfspr(SPRN_LPCR);
	unsigned long new_lpcr;

	if (unlikely(system_state < SYSTEM_RUNNING))
		return index;

	new_lpcr = old_lpcr;
	/* Do not exit powersave upon decrementer as we've setup the timer
	 * offload.
	 */
	new_lpcr &= ~LPCR_PECE1;

	mtspr(SPRN_LPCR, new_lpcr);

	power7_idle_type(PNV_THREAD_SLEEP);

	mtspr(SPRN_LPCR, old_lpcr);

	return index;
}
#endif

static int stop_loop(struct cpuidle_device *dev,
		     struct cpuidle_driver *drv,
		     int index)
{
	power9_idle_type(stop_psscr_table[index].val,
			 stop_psscr_table[index].mask);
	return index;
}

/*
 * States for dedicated partition case.
 */
static struct cpuidle_state powernv_states[CPUIDLE_STATE_MAX] = {
	{ /* Snooze */
		.name = "snooze",
		.desc = "snooze",
		.exit_latency = 0,
		.target_residency = 0,
		.enter = snooze_loop },
};

static int powernv_cpuidle_cpu_online(unsigned int cpu)
{
	struct cpuidle_device *dev = per_cpu(cpuidle_devices, cpu);

	if (dev && cpuidle_get_driver()) {
		cpuidle_pause_and_lock();
		cpuidle_enable_device(dev);
		cpuidle_resume_and_unlock();
	}
	return 0;
}

static int powernv_cpuidle_cpu_dead(unsigned int cpu)
{
	struct cpuidle_device *dev = per_cpu(cpuidle_devices, cpu);

	if (dev && cpuidle_get_driver()) {
		cpuidle_pause_and_lock();
		cpuidle_disable_device(dev);
		cpuidle_resume_and_unlock();
	}
	return 0;
}

/*
 * powernv_cpuidle_driver_init()
 */
static int powernv_cpuidle_driver_init(void)
{
	int idle_state;
	struct cpuidle_driver *drv = &powernv_idle_driver;

	drv->state_count = 0;

	for (idle_state = 0; idle_state < max_idle_state; ++idle_state) {
		/* Is the state not enabled? */
		if (cpuidle_state_table[idle_state].enter == NULL)
			continue;

		drv->states[drv->state_count] =	/* structure copy */
			cpuidle_state_table[idle_state];

		drv->state_count += 1;
	}

	/*
	 * On the PowerNV platform cpu_present may be less than cpu_possible in
	 * cases when firmware detects the CPU, but it is not available to the
	 * OS.  If CONFIG_HOTPLUG_CPU=n, then such CPUs are not hotplugable at
	 * run time and hence cpu_devices are not created for those CPUs by the
	 * generic topology_init().
	 *
	 * drv->cpumask defaults to cpu_possible_mask in
	 * __cpuidle_driver_init().  This breaks cpuidle on PowerNV where
	 * cpu_devices are not created for CPUs in cpu_possible_mask that
	 * cannot be hot-added later at run time.
	 *
	 * Trying cpuidle_register_device() on a CPU without a cpu_device is
	 * incorrect, so pass a correct CPU mask to the generic cpuidle driver.
	 */

	drv->cpumask = (struct cpumask *)cpu_present_mask;

	return 0;
}

static inline void add_powernv_state(int index, const char *name,
				     unsigned int flags,
				     int (*idle_fn)(struct cpuidle_device *,
						    struct cpuidle_driver *,
						    int),
				     unsigned int target_residency,
				     unsigned int exit_latency,
				     u64 psscr_val, u64 psscr_mask)
{
	strlcpy(powernv_states[index].name, name, CPUIDLE_NAME_LEN);
	strlcpy(powernv_states[index].desc, name, CPUIDLE_NAME_LEN);
	powernv_states[index].flags = flags;
	powernv_states[index].target_residency = target_residency;
	powernv_states[index].exit_latency = exit_latency;
	powernv_states[index].enter = idle_fn;
	stop_psscr_table[index].val = psscr_val;
	stop_psscr_table[index].mask = psscr_mask;
}

/*
 * Returns 0 if prop1_len == prop2_len. Else returns -1
 */
static inline int validate_dt_prop_sizes(const char *prop1, int prop1_len,
					 const char *prop2, int prop2_len)
{
	if (prop1_len == prop2_len)
		return 0;

	pr_warn("cpuidle-powernv: array sizes don't match for %s and %s\n",
		prop1, prop2);
	return -1;
}

extern u32 pnv_get_supported_cpuidle_states(void);
static int powernv_add_idle_states(void)
{
	struct device_node *power_mgt;
	int nr_idle_states = 1; /* Snooze */
	int dt_idle_states, count;
	u32 latency_ns[CPUIDLE_STATE_MAX];
	u32 residency_ns[CPUIDLE_STATE_MAX];
	u32 flags[CPUIDLE_STATE_MAX];
	u64 psscr_val[CPUIDLE_STATE_MAX];
	u64 psscr_mask[CPUIDLE_STATE_MAX];
	const char *names[CPUIDLE_STATE_MAX];
	u32 has_stop_states = 0;
	int i, rc;
	u32 supported_flags = pnv_get_supported_cpuidle_states();


	/* Currently we have snooze statically defined */

	power_mgt = of_find_node_by_path("/ibm,opal/power-mgt");
	if (!power_mgt) {
		pr_warn("opal: PowerMgmt Node not found\n");
		goto out;
	}

	/* Read values of any property to determine the num of idle states */
	dt_idle_states = of_property_count_u32_elems(power_mgt, "ibm,cpu-idle-state-flags");
	if (dt_idle_states < 0) {
		pr_warn("cpuidle-powernv: no idle states found in the DT\n");
		goto out;
	}

	count = of_property_count_u32_elems(power_mgt,
					    "ibm,cpu-idle-state-latencies-ns");

	if (validate_dt_prop_sizes("ibm,cpu-idle-state-flags", dt_idle_states,
				   "ibm,cpu-idle-state-latencies-ns",
				   count) != 0)
		goto out;

	count = of_property_count_strings(power_mgt,
					  "ibm,cpu-idle-state-names");
	if (validate_dt_prop_sizes("ibm,cpu-idle-state-flags", dt_idle_states,
				   "ibm,cpu-idle-state-names",
				   count) != 0)
		goto out;

	/*
	 * Since snooze is used as first idle state, max idle states allowed is
	 * CPUIDLE_STATE_MAX -1
	 */
	if (dt_idle_states > CPUIDLE_STATE_MAX - 1) {
		pr_warn("cpuidle-powernv: discovered idle states more than allowed");
		dt_idle_states = CPUIDLE_STATE_MAX - 1;
	}

	if (of_property_read_u32_array(power_mgt,
			"ibm,cpu-idle-state-flags", flags, dt_idle_states)) {
		pr_warn("cpuidle-powernv : missing ibm,cpu-idle-state-flags in DT\n");
		goto out;
	}

	if (of_property_read_u32_array(power_mgt,
		"ibm,cpu-idle-state-latencies-ns", latency_ns,
		dt_idle_states)) {
		pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-latencies-ns in DT\n");
		goto out;
	}
	if (of_property_read_string_array(power_mgt,
		"ibm,cpu-idle-state-names", names, dt_idle_states) < 0) {
		pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-names in DT\n");
		goto out;
	}

	/*
	 * If the idle states use stop instruction, probe for psscr values
	 * and psscr mask which are necessary to specify required stop level.
	 */
	has_stop_states = (flags[0] &
			   (OPAL_PM_STOP_INST_FAST | OPAL_PM_STOP_INST_DEEP));
	if (has_stop_states) {
		count = of_property_count_u64_elems(power_mgt,
						    "ibm,cpu-idle-state-psscr");
		if (validate_dt_prop_sizes("ibm,cpu-idle-state-flags",
					   dt_idle_states,
					   "ibm,cpu-idle-state-psscr",
					   count) != 0)
			goto out;

		count = of_property_count_u64_elems(power_mgt,
						    "ibm,cpu-idle-state-psscr-mask");
		if (validate_dt_prop_sizes("ibm,cpu-idle-state-flags",
					   dt_idle_states,
					   "ibm,cpu-idle-state-psscr-mask",
					   count) != 0)
			goto out;

		if (of_property_read_u64_array(power_mgt,
		    "ibm,cpu-idle-state-psscr", psscr_val, dt_idle_states)) {
			pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-psscr in DT\n");
			goto out;
		}

		if (of_property_read_u64_array(power_mgt,
					       "ibm,cpu-idle-state-psscr-mask",
						psscr_mask, dt_idle_states)) {
			pr_warn("cpuidle-powernv:Missing ibm,cpu-idle-state-psscr-mask in DT\n");
			goto out;
		}
	}

	count = of_property_count_u32_elems(power_mgt,
					    "ibm,cpu-idle-state-residency-ns");

	if (count < 0) {
		rc = count;
	} else if (validate_dt_prop_sizes("ibm,cpu-idle-state-flags",
					  dt_idle_states,
					  "ibm,cpu-idle-state-residency-ns",
					  count) != 0) {
		goto out;
	} else {
		rc = of_property_read_u32_array(power_mgt,
						"ibm,cpu-idle-state-residency-ns",
						residency_ns, dt_idle_states);
	}

	for (i = 0; i < dt_idle_states; i++) {
		unsigned int exit_latency, target_residency;
		bool stops_timebase = false;

		/*
		 * Skip the platform idle state whose flag isn't in
		 * the supported_cpuidle_states flag mask.
		 */
		if ((flags[i] & supported_flags) != flags[i])
			continue;
		/*
		 * If an idle state has exit latency beyond
		 * POWERNV_THRESHOLD_LATENCY_NS then don't use it
		 * in cpu-idle.
		 */
		if (latency_ns[i] > POWERNV_THRESHOLD_LATENCY_NS)
			continue;
		/*
		 * Firmware passes residency and latency values in ns.
		 * cpuidle expects it in us.
		 */
		exit_latency = DIV_ROUND_UP(latency_ns[i], 1000);
		if (!rc)
			target_residency = DIV_ROUND_UP(residency_ns[i], 1000);
		else
			target_residency = 0;

		if (has_stop_states) {
			int err = validate_psscr_val_mask(&psscr_val[i],
							  &psscr_mask[i],
							  flags[i]);
			if (err) {
				report_invalid_psscr_val(psscr_val[i], err);
				continue;
			}
		}

		if (flags[i] & OPAL_PM_TIMEBASE_STOP)
			stops_timebase = true;

		/*
		 * For nap and fastsleep, use default target_residency
		 * values if f/w does not expose it.
		 */
		if (flags[i] & OPAL_PM_NAP_ENABLED) {
			if (!rc)
				target_residency = 100;
			/* Add NAP state */
			add_powernv_state(nr_idle_states, "Nap",
					  CPUIDLE_FLAG_NONE, nap_loop,
					  target_residency, exit_latency, 0, 0);
		} else if (has_stop_states && !stops_timebase) {
			add_powernv_state(nr_idle_states, names[i],
					  CPUIDLE_FLAG_NONE, stop_loop,
					  target_residency, exit_latency,
					  psscr_val[i], psscr_mask[i]);
		}

		/*
		 * All cpuidle states with CPUIDLE_FLAG_TIMER_STOP set must come
		 * within this config dependency check.
		 */
#ifdef CONFIG_TICK_ONESHOT
		else if (flags[i] & OPAL_PM_SLEEP_ENABLED ||
			 flags[i] & OPAL_PM_SLEEP_ENABLED_ER1) {
			if (!rc)
				target_residency = 300000;
			/* Add FASTSLEEP state */
			add_powernv_state(nr_idle_states, "FastSleep",
					  CPUIDLE_FLAG_TIMER_STOP,
					  fastsleep_loop,
					  target_residency, exit_latency, 0, 0);
		} else if (has_stop_states && stops_timebase) {
			add_powernv_state(nr_idle_states, names[i],
					  CPUIDLE_FLAG_TIMER_STOP, stop_loop,
					  target_residency, exit_latency,
					  psscr_val[i], psscr_mask[i]);
		}
#endif
		else
			continue;
		nr_idle_states++;
	}
out:
	return nr_idle_states;
}

/*
 * powernv_idle_probe()
 * Choose state table for shared versus dedicated partition
 */
static int powernv_idle_probe(void)
{
	if (cpuidle_disable != IDLE_NO_OVERRIDE)
		return -ENODEV;

	if (firmware_has_feature(FW_FEATURE_OPAL)) {
		cpuidle_state_table = powernv_states;
		/* Device tree can indicate more idle states */
		max_idle_state = powernv_add_idle_states();
		if (max_idle_state > 1) {
			snooze_timeout_en = true;
			snooze_timeout = powernv_states[1].target_residency *
					 tb_ticks_per_usec;
		}
 	} else
 		return -ENODEV;

	return 0;
}

static int __init powernv_processor_idle_init(void)
{
	int retval;

	retval = powernv_idle_probe();
	if (retval)
		return retval;

	powernv_cpuidle_driver_init();
	retval = cpuidle_register(&powernv_idle_driver, NULL);
	if (retval) {
		printk(KERN_DEBUG "Registration of powernv driver failed.\n");
		return retval;
	}

	retval = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
					   "cpuidle/powernv:online",
					   powernv_cpuidle_cpu_online, NULL);
	WARN_ON(retval < 0);
	retval = cpuhp_setup_state_nocalls(CPUHP_CPUIDLE_DEAD,
					   "cpuidle/powernv:dead", NULL,
					   powernv_cpuidle_cpu_dead);
	WARN_ON(retval < 0);
	printk(KERN_DEBUG "powernv_idle_driver registered\n");
	return 0;
}

device_initcall(powernv_processor_idle_init);
Commit	Line	Data
b2441318	1	// SPDX-License-Identifier: GPL-2.0
2c2e6ecf DD	2	/*
	3	* cpuidle-powernv - idle state cpuidle driver.
	4	* Adapted from drivers/cpuidle/cpuidle-pseries
	5	*
	6	*/
	7
	8	#include <linux/kernel.h>
	9	#include <linux/module.h>
	10	#include <linux/init.h>
	11	#include <linux/moduleparam.h>
	12	#include <linux/cpuidle.h>
	13	#include <linux/cpu.h>
	14	#include <linux/notifier.h>
0d948730	15	#include <linux/clockchips.h>
0888839c	16	#include <linux/of.h>
92c83ff5	17	#include <linux/slab.h>
2c2e6ecf DD	18
	19	#include <asm/machdep.h>
	20	#include <asm/firmware.h>
8eb8ac89	21	#include <asm/opal.h>
591ac0cb	22	#include <asm/runlatch.h>
09206b60	23	#include <asm/cpuidle.h>
2c2e6ecf	24
9e9fc6f0 GS	25	/*
	26	* Expose only those Hardware idle states via the cpuidle framework
	27	* that have latency value below POWERNV_THRESHOLD_LATENCY_NS.
	28	*/
3005c597 SP	29	#define POWERNV_THRESHOLD_LATENCY_NS 200000
3005c597 SP	30
ed61390b	31	static struct cpuidle_driver powernv_idle_driver = {
2c2e6ecf DD	32	.name = "powernv_idle",
	33	.owner = THIS_MODULE,
	34	};
	35
624e46d0 NP	36	static int max_idle_state __read_mostly;
624e46d0 NP	37	static struct cpuidle_state *cpuidle_state_table __read_mostly;
3005c597	38
09206b60 GS	39	struct stop_psscr_table {
	40	u64 val;
	41	u64 mask;
	42	};
	43
624e46d0	44	static struct stop_psscr_table stop_psscr_table[CPUIDLE_STATE_MAX] __read_mostly;
3005c597	45
624e46d0 NP	46	static u64 snooze_timeout __read_mostly;
624e46d0 NP	47	static bool snooze_timeout_en __read_mostly;
2c2e6ecf DD	48
	49	static int snooze_loop(struct cpuidle_device *dev,
	50	struct cpuidle_driver *drv,
	51	int index)
	52	{
78eaa10f SB	53	u64 snooze_exit_time;
78eaa10f SB	54
2c2e6ecf DD	55	set_thread_flag(TIF_POLLING_NRFLAG);
2c2e6ecf DD	56
3fc5ee92 NP	57	local_irq_enable();
3fc5ee92 NP	58
78eaa10f	59	snooze_exit_time = get_tb() + snooze_timeout;
591ac0cb	60	ppc64_runlatch_off();
26eb48a9	61	HMT_very_low();
2c2e6ecf	62	while (!need_resched()) {
7ded4291 NP	63	if (likely(snooze_timeout_en) && get_tb() > snooze_exit_time) {
	64	/*
	65	* Task has not woken up but we are exiting the polling
	66	* loop anyway. Require a barrier after polling is
	67	* cleared to order subsequent test of need_resched().
	68	*/
	69	clear_thread_flag(TIF_POLLING_NRFLAG);
	70	smp_mb();
78eaa10f	71	break;
7ded4291	72	}
2c2e6ecf DD	73	}
	74
	75	HMT_medium();
591ac0cb	76	ppc64_runlatch_on();
2c2e6ecf	77	clear_thread_flag(TIF_POLLING_NRFLAG);
3fc5ee92	78
f1343d04 NP	79	local_irq_disable();
f1343d04 NP	80
2c2e6ecf DD	81	return index;
	82	}
	83
	84	static int nap_loop(struct cpuidle_device *dev,
	85	struct cpuidle_driver *drv,
	86	int index)
	87	{
2201f994 NP	88	power7_idle_type(PNV_THREAD_NAP);
2201f994 NP	89
2c2e6ecf DD	90	return index;
	91	}
	92
cc5a2f7b	93	/* Register for fastsleep only in oneshot mode of broadcast */
cc5a2f7b	94	#ifdef CONFIG_TICK_ONESHOT
0d948730 PM	95	static int fastsleep_loop(struct cpuidle_device *dev,
	96	struct cpuidle_driver *drv,
	97	int index)
	98	{
	99	unsigned long old_lpcr = mfspr(SPRN_LPCR);
	100	unsigned long new_lpcr;
	101
	102	if (unlikely(system_state < SYSTEM_RUNNING))
	103	return index;
	104
	105	new_lpcr = old_lpcr;
9b6a68d9 MN	106	/* Do not exit powersave upon decrementer as we've setup the timer
9b6a68d9 MN	107	* offload.
0d948730	108	*/
9b6a68d9	109	new_lpcr &= ~LPCR_PECE1;
0d948730 PM	110
0d948730 PM	111	mtspr(SPRN_LPCR, new_lpcr);
2201f994 NP	112
2201f994 NP	113	power7_idle_type(PNV_THREAD_SLEEP);
0d948730 PM	114
	115	mtspr(SPRN_LPCR, old_lpcr);
	116
	117	return index;
	118	}
cc5a2f7b	119	#endif
3005c597 SP	120
	121	static int stop_loop(struct cpuidle_device *dev,
	122	struct cpuidle_driver *drv,
	123	int index)
	124	{
2201f994	125	power9_idle_type(stop_psscr_table[index].val,
09206b60	126	stop_psscr_table[index].mask);
3005c597 SP	127	return index;
	128	}
	129
2c2e6ecf DD	130	/*
	131	* States for dedicated partition case.
	132	*/
169f3fae	133	static struct cpuidle_state powernv_states[CPUIDLE_STATE_MAX] = {
2c2e6ecf DD	134	{ /* Snooze */
	135	.name = "snooze",
	136	.desc = "snooze",
2c2e6ecf DD	137	.exit_latency = 0,
2c2e6ecf DD	138	.target_residency = 0,
957efced	139	.enter = snooze_loop },
2c2e6ecf DD	140	};
2c2e6ecf DD	141
10fcca9d	142	static int powernv_cpuidle_cpu_online(unsigned int cpu)
2c2e6ecf	143	{
10fcca9d	144	struct cpuidle_device *dev = per_cpu(cpuidle_devices, cpu);
2c2e6ecf DD	145
2c2e6ecf DD	146	if (dev && cpuidle_get_driver()) {
10fcca9d SAS	147	cpuidle_pause_and_lock();
	148	cpuidle_enable_device(dev);
	149	cpuidle_resume_and_unlock();
	150	}
	151	return 0;
	152	}
2c2e6ecf	153
10fcca9d SAS	154	static int powernv_cpuidle_cpu_dead(unsigned int cpu)
	155	{
	156	struct cpuidle_device *dev = per_cpu(cpuidle_devices, cpu);
2c2e6ecf	157
10fcca9d SAS	158	if (dev && cpuidle_get_driver()) {
	159	cpuidle_pause_and_lock();
	160	cpuidle_disable_device(dev);
	161	cpuidle_resume_and_unlock();
2c2e6ecf	162	}
10fcca9d	163	return 0;
2c2e6ecf DD	164	}
2c2e6ecf DD	165
2c2e6ecf DD	166	/*
	167	* powernv_cpuidle_driver_init()
	168	*/
	169	static int powernv_cpuidle_driver_init(void)
	170	{
	171	int idle_state;
	172	struct cpuidle_driver *drv = &powernv_idle_driver;
	173
	174	drv->state_count = 0;
	175
	176	for (idle_state = 0; idle_state < max_idle_state; ++idle_state) {
	177	/* Is the state not enabled? */
	178	if (cpuidle_state_table[idle_state].enter == NULL)
	179	continue;
	180
	181	drv->states[drv->state_count] = /* structure copy */
	182	cpuidle_state_table[idle_state];
	183
	184	drv->state_count += 1;
	185	}
	186
293d264f VS	187	/*
	188	* On the PowerNV platform cpu_present may be less than cpu_possible in
	189	* cases when firmware detects the CPU, but it is not available to the
	190	* OS. If CONFIG_HOTPLUG_CPU=n, then such CPUs are not hotplugable at
	191	* run time and hence cpu_devices are not created for those CPUs by the
	192	* generic topology_init().
	193	*
	194	* drv->cpumask defaults to cpu_possible_mask in
	195	* __cpuidle_driver_init(). This breaks cpuidle on PowerNV where
	196	* cpu_devices are not created for CPUs in cpu_possible_mask that
	197	* cannot be hot-added later at run time.
	198	*
	199	* Trying cpuidle_register_device() on a CPU without a cpu_device is
	200	* incorrect, so pass a correct CPU mask to the generic cpuidle driver.
	201	*/
	202
	203	drv->cpumask = (struct cpumask *)cpu_present_mask;
	204
2c2e6ecf DD	205	return 0;
	206	}
	207
9e9fc6f0 GS	208	static inline void add_powernv_state(int index, const char *name,
	209	unsigned int flags,
	210	int (idle_fn)(struct cpuidle_device ,
	211	struct cpuidle_driver *,
	212	int),
	213	unsigned int target_residency,
	214	unsigned int exit_latency,
09206b60	215	u64 psscr_val, u64 psscr_mask)
9e9fc6f0 GS	216	{
	217	strlcpy(powernv_states[index].name, name, CPUIDLE_NAME_LEN);
	218	strlcpy(powernv_states[index].desc, name, CPUIDLE_NAME_LEN);
	219	powernv_states[index].flags = flags;
	220	powernv_states[index].target_residency = target_residency;
	221	powernv_states[index].exit_latency = exit_latency;
	222	powernv_states[index].enter = idle_fn;
09206b60 GS	223	stop_psscr_table[index].val = psscr_val;
09206b60 GS	224	stop_psscr_table[index].mask = psscr_mask;
9e9fc6f0 GS	225	}
9e9fc6f0 GS	226
ecad4502 GS	227	/*
	228	* Returns 0 if prop1_len == prop2_len. Else returns -1
	229	*/
	230	static inline int validate_dt_prop_sizes(const char *prop1, int prop1_len,
	231	const char *prop2, int prop2_len)
	232	{
	233	if (prop1_len == prop2_len)
	234	return 0;
	235
	236	pr_warn("cpuidle-powernv: array sizes don't match for %s and %s\n",
	237	prop1, prop2);
	238	return -1;
	239	}
	240
785a12af	241	extern u32 pnv_get_supported_cpuidle_states(void);
0888839c PM	242	static int powernv_add_idle_states(void)
	243	{
	244	struct device_node *power_mgt;
0888839c	245	int nr_idle_states = 1; /* Snooze */
ecad4502	246	int dt_idle_states, count;
957efced SP	247	u32 latency_ns[CPUIDLE_STATE_MAX];
	248	u32 residency_ns[CPUIDLE_STATE_MAX];
	249	u32 flags[CPUIDLE_STATE_MAX];
3005c597	250	u64 psscr_val[CPUIDLE_STATE_MAX];
09206b60	251	u64 psscr_mask[CPUIDLE_STATE_MAX];
3005c597	252	const char *names[CPUIDLE_STATE_MAX];
09206b60	253	u32 has_stop_states = 0;
92c83ff5	254	int i, rc;
785a12af GS	255	u32 supported_flags = pnv_get_supported_cpuidle_states();
785a12af GS	256
0888839c PM	257
	258	/* Currently we have snooze statically defined */
	259
	260	power_mgt = of_find_node_by_path("/ibm,opal/power-mgt");
	261	if (!power_mgt) {
	262	pr_warn("opal: PowerMgmt Node not found\n");
92c83ff5	263	goto out;
0888839c PM	264	}
0888839c PM	265
70734a78 PM	266	/* Read values of any property to determine the num of idle states */
	267	dt_idle_states = of_property_count_u32_elems(power_mgt, "ibm,cpu-idle-state-flags");
	268	if (dt_idle_states < 0) {
	269	pr_warn("cpuidle-powernv: no idle states found in the DT\n");
92c83ff5	270	goto out;
0888839c PM	271	}
0888839c PM	272
ecad4502 GS	273	count = of_property_count_u32_elems(power_mgt,
	274	"ibm,cpu-idle-state-latencies-ns");
	275
	276	if (validate_dt_prop_sizes("ibm,cpu-idle-state-flags", dt_idle_states,
	277	"ibm,cpu-idle-state-latencies-ns",
	278	count) != 0)
	279	goto out;
	280
	281	count = of_property_count_strings(power_mgt,
	282	"ibm,cpu-idle-state-names");
	283	if (validate_dt_prop_sizes("ibm,cpu-idle-state-flags", dt_idle_states,
	284	"ibm,cpu-idle-state-names",
	285	count) != 0)
	286	goto out;
	287
957efced SP	288	/*
	289	* Since snooze is used as first idle state, max idle states allowed is
	290	* CPUIDLE_STATE_MAX -1
	291	*/
	292	if (dt_idle_states > CPUIDLE_STATE_MAX - 1) {
	293	pr_warn("cpuidle-powernv: discovered idle states more than allowed");
	294	dt_idle_states = CPUIDLE_STATE_MAX - 1;
	295	}
	296
70734a78 PM	297	if (of_property_read_u32_array(power_mgt,
	298	"ibm,cpu-idle-state-flags", flags, dt_idle_states)) {
	299	pr_warn("cpuidle-powernv : missing ibm,cpu-idle-state-flags in DT\n");
957efced	300	goto out;
70734a78	301	}
92c83ff5	302
957efced SP	303	if (of_property_read_u32_array(power_mgt,
	304	"ibm,cpu-idle-state-latencies-ns", latency_ns,
	305	dt_idle_states)) {
92c83ff5	306	pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-latencies-ns in DT\n");
957efced	307	goto out;
74aa51b5	308	}
3005c597 SP	309	if (of_property_read_string_array(power_mgt,
	310	"ibm,cpu-idle-state-names", names, dt_idle_states) < 0) {
	311	pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-names in DT\n");
	312	goto out;
	313	}
	314
	315	/*
	316	* If the idle states use stop instruction, probe for psscr values
09206b60	317	* and psscr mask which are necessary to specify required stop level.
3005c597	318	*/
09206b60 GS	319	has_stop_states = (flags[0] &
	320	(OPAL_PM_STOP_INST_FAST \| OPAL_PM_STOP_INST_DEEP));
	321	if (has_stop_states) {
ecad4502 GS	322	count = of_property_count_u64_elems(power_mgt,
	323	"ibm,cpu-idle-state-psscr");
	324	if (validate_dt_prop_sizes("ibm,cpu-idle-state-flags",
	325	dt_idle_states,
	326	"ibm,cpu-idle-state-psscr",
	327	count) != 0)
	328	goto out;
	329
	330	count = of_property_count_u64_elems(power_mgt,
	331	"ibm,cpu-idle-state-psscr-mask");
	332	if (validate_dt_prop_sizes("ibm,cpu-idle-state-flags",
	333	dt_idle_states,
	334	"ibm,cpu-idle-state-psscr-mask",
	335	count) != 0)
	336	goto out;
	337
3005c597 SP	338	if (of_property_read_u64_array(power_mgt,
3005c597 SP	339	"ibm,cpu-idle-state-psscr", psscr_val, dt_idle_states)) {
09206b60	340	pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-psscr in DT\n");
3005c597 SP	341	goto out;
3005c597 SP	342	}
74aa51b5	343
09206b60 GS	344	if (of_property_read_u64_array(power_mgt,
	345	"ibm,cpu-idle-state-psscr-mask",
	346	psscr_mask, dt_idle_states)) {
	347	pr_warn("cpuidle-powernv:Missing ibm,cpu-idle-state-psscr-mask in DT\n");
	348	goto out;
	349	}
	350	}
	351
ecad4502 GS	352	count = of_property_count_u32_elems(power_mgt,
	353	"ibm,cpu-idle-state-residency-ns");
	354
	355	if (count < 0) {
	356	rc = count;
	357	} else if (validate_dt_prop_sizes("ibm,cpu-idle-state-flags",
	358	dt_idle_states,
	359	"ibm,cpu-idle-state-residency-ns",
	360	count) != 0) {
	361	goto out;
	362	} else {
	363	rc = of_property_read_u32_array(power_mgt,
	364	"ibm,cpu-idle-state-residency-ns",
	365	residency_ns, dt_idle_states);
	366	}
0888839c PM	367
0888839c PM	368	for (i = 0; i < dt_idle_states; i++) {
9e9fc6f0	369	unsigned int exit_latency, target_residency;
f9122ee4	370	bool stops_timebase = false;
785a12af GS	371
	372	/*
	373	* Skip the platform idle state whose flag isn't in
	374	* the supported_cpuidle_states flag mask.
	375	*/
	376	if ((flags[i] & supported_flags) != flags[i])
	377	continue;
3005c597 SP	378	/*
	379	* If an idle state has exit latency beyond
	380	* POWERNV_THRESHOLD_LATENCY_NS then don't use it
	381	* in cpu-idle.
	382	*/
	383	if (latency_ns[i] > POWERNV_THRESHOLD_LATENCY_NS)
	384	continue;
9e9fc6f0 GS	385	/*
	386	* Firmware passes residency and latency values in ns.
	387	* cpuidle expects it in us.
	388	*/
8d4e10e9	389	exit_latency = DIV_ROUND_UP(latency_ns[i], 1000);
9e9fc6f0	390	if (!rc)
8d4e10e9	391	target_residency = DIV_ROUND_UP(residency_ns[i], 1000);
9e9fc6f0 GS	392	else
9e9fc6f0 GS	393	target_residency = 0;
0888839c	394
09206b60 GS	395	if (has_stop_states) {
	396	int err = validate_psscr_val_mask(&psscr_val[i],
	397	&psscr_mask[i],
	398	flags[i]);
	399	if (err) {
	400	report_invalid_psscr_val(psscr_val[i], err);
	401	continue;
	402	}
	403	}
	404
f9122ee4 GS	405	if (flags[i] & OPAL_PM_TIMEBASE_STOP)
	406	stops_timebase = true;
	407
92c83ff5	408	/*
9e9fc6f0 GS	409	* For nap and fastsleep, use default target_residency
9e9fc6f0 GS	410	* values if f/w does not expose it.
74aa51b5	411	*/
70734a78	412	if (flags[i] & OPAL_PM_NAP_ENABLED) {
9e9fc6f0 GS	413	if (!rc)
9e9fc6f0 GS	414	target_residency = 100;
0888839c	415	/* Add NAP state */
9e9fc6f0 GS	416	add_powernv_state(nr_idle_states, "Nap",
9e9fc6f0 GS	417	CPUIDLE_FLAG_NONE, nap_loop,
09206b60	418	target_residency, exit_latency, 0, 0);
f9122ee4	419	} else if (has_stop_states && !stops_timebase) {
9e9fc6f0 GS	420	add_powernv_state(nr_idle_states, names[i],
	421	CPUIDLE_FLAG_NONE, stop_loop,
	422	target_residency, exit_latency,
09206b60	423	psscr_val[i], psscr_mask[i]);
cc5a2f7b	424	}
	425
	426	/*
	427	* All cpuidle states with CPUIDLE_FLAG_TIMER_STOP set must come
	428	* within this config dependency check.
	429	*/
	430	#ifdef CONFIG_TICK_ONESHOT
f9122ee4 GS	431	else if (flags[i] & OPAL_PM_SLEEP_ENABLED \|\|
f9122ee4 GS	432	flags[i] & OPAL_PM_SLEEP_ENABLED_ER1) {
9e9fc6f0 GS	433	if (!rc)
9e9fc6f0 GS	434	target_residency = 300000;
0888839c	435	/* Add FASTSLEEP state */
9e9fc6f0 GS	436	add_powernv_state(nr_idle_states, "FastSleep",
	437	CPUIDLE_FLAG_TIMER_STOP,
	438	fastsleep_loop,
09206b60	439	target_residency, exit_latency, 0, 0);
f9122ee4	440	} else if (has_stop_states && stops_timebase) {
9e9fc6f0 GS	441	add_powernv_state(nr_idle_states, names[i],
	442	CPUIDLE_FLAG_TIMER_STOP, stop_loop,
	443	target_residency, exit_latency,
09206b60	444	psscr_val[i], psscr_mask[i]);
0888839c	445	}
cc5a2f7b	446	#endif
f9122ee4 GS	447	else
f9122ee4 GS	448	continue;
92c83ff5	449	nr_idle_states++;
0888839c	450	}
92c83ff5	451	out:
0888839c PM	452	return nr_idle_states;
	453	}
	454
2c2e6ecf DD	455	/*
	456	* powernv_idle_probe()
	457	* Choose state table for shared versus dedicated partition
	458	*/
	459	static int powernv_idle_probe(void)
	460	{
2c2e6ecf DD	461	if (cpuidle_disable != IDLE_NO_OVERRIDE)
	462	return -ENODEV;
	463
e4d54f71	464	if (firmware_has_feature(FW_FEATURE_OPAL)) {
2c2e6ecf	465	cpuidle_state_table = powernv_states;
0888839c PM	466	/* Device tree can indicate more idle states */
0888839c PM	467	max_idle_state = powernv_add_idle_states();
78eaa10f SB	468	if (max_idle_state > 1) {
	469	snooze_timeout_en = true;
	470	snooze_timeout = powernv_states[1].target_residency *
	471	tb_ticks_per_usec;
	472	}
2c2e6ecf DD	473	} else
	474	return -ENODEV;
	475
	476	return 0;
	477	}
	478
	479	static int __init powernv_processor_idle_init(void)
	480	{
	481	int retval;
	482
	483	retval = powernv_idle_probe();
	484	if (retval)
	485	return retval;
	486
	487	powernv_cpuidle_driver_init();
	488	retval = cpuidle_register(&powernv_idle_driver, NULL);
	489	if (retval) {
	490	printk(KERN_DEBUG "Registration of powernv driver failed.\n");
	491	return retval;
	492	}
	493
10fcca9d SAS	494	retval = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
	495	"cpuidle/powernv:online",
	496	powernv_cpuidle_cpu_online, NULL);
	497	WARN_ON(retval < 0);
	498	retval = cpuhp_setup_state_nocalls(CPUHP_CPUIDLE_DEAD,
	499	"cpuidle/powernv:dead", NULL,
	500	powernv_cpuidle_cpu_dead);
	501	WARN_ON(retval < 0);
2c2e6ecf DD	502	printk(KERN_DEBUG "powernv_idle_driver registered\n");
	503	return 0;
	504	}
	505
	506	device_initcall(powernv_processor_idle_init);