[linux.git] / drivers / cpuidle / cpuidle-powernv.c

// SPDX-License-Identifier: GPL-2.0
/*
 *  cpuidle-powernv - idle state cpuidle driver.
 *  Adapted from drivers/cpuidle/cpuidle-pseries
 *
 */

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/moduleparam.h>
#include <linux/cpuidle.h>
#include <linux/cpu.h>
#include <linux/notifier.h>
#include <linux/clockchips.h>
#include <linux/of.h>
#include <linux/slab.h>

#include <asm/machdep.h>
#include <asm/firmware.h>
#include <asm/opal.h>
#include <asm/runlatch.h>
#include <asm/cpuidle.h>

/*
 * Expose only those Hardware idle states via the cpuidle framework
 * that have latency value below POWERNV_THRESHOLD_LATENCY_NS.
 */
#define POWERNV_THRESHOLD_LATENCY_NS 200000

static struct cpuidle_driver powernv_idle_driver = {
	.name             = "powernv_idle",
	.owner            = THIS_MODULE,
};

static int max_idle_state __read_mostly;
static struct cpuidle_state *cpuidle_state_table __read_mostly;

struct stop_psscr_table {
	u64 val;
	u64 mask;
};

static struct stop_psscr_table stop_psscr_table[CPUIDLE_STATE_MAX] __read_mostly;

static u64 default_snooze_timeout __read_mostly;
static bool snooze_timeout_en __read_mostly;

static u64 get_snooze_timeout(struct cpuidle_device *dev,
			      struct cpuidle_driver *drv,
			      int index)
{
	int i;

	if (unlikely(!snooze_timeout_en))
		return default_snooze_timeout;

	for (i = index + 1; i < drv->state_count; i++) {
		struct cpuidle_state *s = &drv->states[i];
		struct cpuidle_state_usage *su = &dev->states_usage[i];

		if (s->disabled || su->disable)
			continue;

		return s->target_residency * tb_ticks_per_usec;
	}

	return default_snooze_timeout;
}

static int snooze_loop(struct cpuidle_device *dev,
			struct cpuidle_driver *drv,
			int index)
{
	u64 snooze_exit_time;

	set_thread_flag(TIF_POLLING_NRFLAG);

	local_irq_enable();

	snooze_exit_time = get_tb() + get_snooze_timeout(dev, drv, index);
	ppc64_runlatch_off();
	HMT_very_low();
	while (!need_resched()) {
		if (likely(snooze_timeout_en) && get_tb() > snooze_exit_time) {
			/*
			 * Task has not woken up but we are exiting the polling
			 * loop anyway. Require a barrier after polling is
			 * cleared to order subsequent test of need_resched().
			 */
			clear_thread_flag(TIF_POLLING_NRFLAG);
			smp_mb();
			break;
		}
	}

	HMT_medium();
	ppc64_runlatch_on();
	clear_thread_flag(TIF_POLLING_NRFLAG);

	local_irq_disable();

	return index;
}

static int nap_loop(struct cpuidle_device *dev,
			struct cpuidle_driver *drv,
			int index)
{
	power7_idle_type(PNV_THREAD_NAP);

	return index;
}

/* Register for fastsleep only in oneshot mode of broadcast */
#ifdef CONFIG_TICK_ONESHOT
static int fastsleep_loop(struct cpuidle_device *dev,
				struct cpuidle_driver *drv,
				int index)
{
	unsigned long old_lpcr = mfspr(SPRN_LPCR);
	unsigned long new_lpcr;

	if (unlikely(system_state < SYSTEM_RUNNING))
		return index;

	new_lpcr = old_lpcr;
	/* Do not exit powersave upon decrementer as we've setup the timer
	 * offload.
	 */
	new_lpcr &= ~LPCR_PECE1;

	mtspr(SPRN_LPCR, new_lpcr);

	power7_idle_type(PNV_THREAD_SLEEP);

	mtspr(SPRN_LPCR, old_lpcr);

	return index;
}
#endif

static int stop_loop(struct cpuidle_device *dev,
		     struct cpuidle_driver *drv,
		     int index)
{
	power9_idle_type(stop_psscr_table[index].val,
			 stop_psscr_table[index].mask);
	return index;
}

/*
 * States for dedicated partition case.
 */
static struct cpuidle_state powernv_states[CPUIDLE_STATE_MAX] = {
	{ /* Snooze */
		.name = "snooze",
		.desc = "snooze",
		.exit_latency = 0,
		.target_residency = 0,
		.enter = snooze_loop },
};

static int powernv_cpuidle_cpu_online(unsigned int cpu)
{
	struct cpuidle_device *dev = per_cpu(cpuidle_devices, cpu);

	if (dev && cpuidle_get_driver()) {
		cpuidle_pause_and_lock();
		cpuidle_enable_device(dev);
		cpuidle_resume_and_unlock();
	}
	return 0;
}

static int powernv_cpuidle_cpu_dead(unsigned int cpu)
{
	struct cpuidle_device *dev = per_cpu(cpuidle_devices, cpu);

	if (dev && cpuidle_get_driver()) {
		cpuidle_pause_and_lock();
		cpuidle_disable_device(dev);
		cpuidle_resume_and_unlock();
	}
	return 0;
}

/*
 * powernv_cpuidle_driver_init()
 */
static int powernv_cpuidle_driver_init(void)
{
	int idle_state;
	struct cpuidle_driver *drv = &powernv_idle_driver;

	drv->state_count = 0;

	for (idle_state = 0; idle_state < max_idle_state; ++idle_state) {
		/* Is the state not enabled? */
		if (cpuidle_state_table[idle_state].enter == NULL)
			continue;

		drv->states[drv->state_count] =	/* structure copy */
			cpuidle_state_table[idle_state];

		drv->state_count += 1;
	}

	/*
	 * On the PowerNV platform cpu_present may be less than cpu_possible in
	 * cases when firmware detects the CPU, but it is not available to the
	 * OS.  If CONFIG_HOTPLUG_CPU=n, then such CPUs are not hotplugable at
	 * run time and hence cpu_devices are not created for those CPUs by the
	 * generic topology_init().
	 *
	 * drv->cpumask defaults to cpu_possible_mask in
	 * __cpuidle_driver_init().  This breaks cpuidle on PowerNV where
	 * cpu_devices are not created for CPUs in cpu_possible_mask that
	 * cannot be hot-added later at run time.
	 *
	 * Trying cpuidle_register_device() on a CPU without a cpu_device is
	 * incorrect, so pass a correct CPU mask to the generic cpuidle driver.
	 */

	drv->cpumask = (struct cpumask *)cpu_present_mask;

	return 0;
}

static inline void add_powernv_state(int index, const char *name,
				     unsigned int flags,
				     int (*idle_fn)(struct cpuidle_device *,
						    struct cpuidle_driver *,
						    int),
				     unsigned int target_residency,
				     unsigned int exit_latency,
				     u64 psscr_val, u64 psscr_mask)
{
	strlcpy(powernv_states[index].name, name, CPUIDLE_NAME_LEN);
	strlcpy(powernv_states[index].desc, name, CPUIDLE_NAME_LEN);
	powernv_states[index].flags = flags;
	powernv_states[index].target_residency = target_residency;
	powernv_states[index].exit_latency = exit_latency;
	powernv_states[index].enter = idle_fn;
	/* For power8 and below psscr_* will be 0 */
	stop_psscr_table[index].val = psscr_val;
	stop_psscr_table[index].mask = psscr_mask;
}

/*
 * Returns 0 if prop1_len == prop2_len. Else returns -1
 */
static inline int validate_dt_prop_sizes(const char *prop1, int prop1_len,
					 const char *prop2, int prop2_len)
{
	if (prop1_len == prop2_len)
		return 0;

	pr_warn("cpuidle-powernv: array sizes don't match for %s and %s\n",
		prop1, prop2);
	return -1;
}

extern u32 pnv_get_supported_cpuidle_states(void);
static int powernv_add_idle_states(void)
{
	int nr_idle_states = 1; /* Snooze */
	int dt_idle_states;
	u32 has_stop_states = 0;
	int i;
	u32 supported_flags = pnv_get_supported_cpuidle_states();


	/* Currently we have snooze statically defined */
	if (nr_pnv_idle_states <= 0) {
		pr_warn("cpuidle-powernv : Only Snooze is available\n");
		goto out;
	}

	/* TODO: Count only states which are eligible for cpuidle */
	dt_idle_states = nr_pnv_idle_states;

	/*
	 * Since snooze is used as first idle state, max idle states allowed is
	 * CPUIDLE_STATE_MAX -1
	 */
	if (nr_pnv_idle_states > CPUIDLE_STATE_MAX - 1) {
		pr_warn("cpuidle-powernv: discovered idle states more than allowed");
		dt_idle_states = CPUIDLE_STATE_MAX - 1;
	}

	/*
	 * If the idle states use stop instruction, probe for psscr values
	 * and psscr mask which are necessary to specify required stop level.
	 */
	has_stop_states = (pnv_idle_states[0].flags &
			   (OPAL_PM_STOP_INST_FAST | OPAL_PM_STOP_INST_DEEP));

	for (i = 0; i < dt_idle_states; i++) {
		unsigned int exit_latency, target_residency;
		bool stops_timebase = false;
		struct pnv_idle_states_t *state = &pnv_idle_states[i];

		/*
		 * Skip the platform idle state whose flag isn't in
		 * the supported_cpuidle_states flag mask.
		 */
		if ((state->flags & supported_flags) != state->flags)
			continue;
		/*
		 * If an idle state has exit latency beyond
		 * POWERNV_THRESHOLD_LATENCY_NS then don't use it
		 * in cpu-idle.
		 */
		if (state->latency_ns > POWERNV_THRESHOLD_LATENCY_NS)
			continue;
		/*
		 * Firmware passes residency and latency values in ns.
		 * cpuidle expects it in us.
		 */
		exit_latency = DIV_ROUND_UP(state->latency_ns, 1000);
		target_residency = DIV_ROUND_UP(state->residency_ns, 1000);

		if (has_stop_states && !(state->valid))
				continue;

		if (state->flags & OPAL_PM_TIMEBASE_STOP)
			stops_timebase = true;

		if (state->flags & OPAL_PM_NAP_ENABLED) {
			/* Add NAP state */
			add_powernv_state(nr_idle_states, "Nap",
					  CPUIDLE_FLAG_NONE, nap_loop,
					  target_residency, exit_latency, 0, 0);
		} else if (has_stop_states && !stops_timebase) {
			add_powernv_state(nr_idle_states, state->name,
					  CPUIDLE_FLAG_NONE, stop_loop,
					  target_residency, exit_latency,
					  state->psscr_val,
					  state->psscr_mask);
		}

		/*
		 * All cpuidle states with CPUIDLE_FLAG_TIMER_STOP set must come
		 * within this config dependency check.
		 */
#ifdef CONFIG_TICK_ONESHOT
		else if (state->flags & OPAL_PM_SLEEP_ENABLED ||
			 state->flags & OPAL_PM_SLEEP_ENABLED_ER1) {
			/* Add FASTSLEEP state */
			add_powernv_state(nr_idle_states, "FastSleep",
					  CPUIDLE_FLAG_TIMER_STOP,
					  fastsleep_loop,
					  target_residency, exit_latency, 0, 0);
		} else if (has_stop_states && stops_timebase) {
			add_powernv_state(nr_idle_states, state->name,
					  CPUIDLE_FLAG_TIMER_STOP, stop_loop,
					  target_residency, exit_latency,
					  state->psscr_val,
					  state->psscr_mask);
		}
#endif
		else
			continue;
		nr_idle_states++;
	}
out:
	return nr_idle_states;
}

/*
 * powernv_idle_probe()
 * Choose state table for shared versus dedicated partition
 */
static int powernv_idle_probe(void)
{
	if (cpuidle_disable != IDLE_NO_OVERRIDE)
		return -ENODEV;

	if (firmware_has_feature(FW_FEATURE_OPAL)) {
		cpuidle_state_table = powernv_states;
		/* Device tree can indicate more idle states */
		max_idle_state = powernv_add_idle_states();
		default_snooze_timeout = TICK_USEC * tb_ticks_per_usec;
		if (max_idle_state > 1)
			snooze_timeout_en = true;
 	} else
 		return -ENODEV;

	return 0;
}

static int __init powernv_processor_idle_init(void)
{
	int retval;

	retval = powernv_idle_probe();
	if (retval)
		return retval;

	powernv_cpuidle_driver_init();
	retval = cpuidle_register(&powernv_idle_driver, NULL);
	if (retval) {
		printk(KERN_DEBUG "Registration of powernv driver failed.\n");
		return retval;
	}

	retval = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
					   "cpuidle/powernv:online",
					   powernv_cpuidle_cpu_online, NULL);
	WARN_ON(retval < 0);
	retval = cpuhp_setup_state_nocalls(CPUHP_CPUIDLE_DEAD,
					   "cpuidle/powernv:dead", NULL,
					   powernv_cpuidle_cpu_dead);
	WARN_ON(retval < 0);
	printk(KERN_DEBUG "powernv_idle_driver registered\n");
	return 0;
}

device_initcall(powernv_processor_idle_init);
Commit	Line	Data
b2441318	1	// SPDX-License-Identifier: GPL-2.0
2c2e6ecf DD	2	/*
	3	* cpuidle-powernv - idle state cpuidle driver.
	4	* Adapted from drivers/cpuidle/cpuidle-pseries
	5	*
	6	*/
	7
	8	#include <linux/kernel.h>
	9	#include <linux/module.h>
	10	#include <linux/init.h>
	11	#include <linux/moduleparam.h>
	12	#include <linux/cpuidle.h>
	13	#include <linux/cpu.h>
	14	#include <linux/notifier.h>
0d948730	15	#include <linux/clockchips.h>
0888839c	16	#include <linux/of.h>
92c83ff5	17	#include <linux/slab.h>
2c2e6ecf DD	18
	19	#include <asm/machdep.h>
	20	#include <asm/firmware.h>
8eb8ac89	21	#include <asm/opal.h>
591ac0cb	22	#include <asm/runlatch.h>
09206b60	23	#include <asm/cpuidle.h>
2c2e6ecf	24
9e9fc6f0 GS	25	/*
	26	* Expose only those Hardware idle states via the cpuidle framework
	27	* that have latency value below POWERNV_THRESHOLD_LATENCY_NS.
	28	*/
3005c597 SP	29	#define POWERNV_THRESHOLD_LATENCY_NS 200000
3005c597 SP	30
ed61390b	31	static struct cpuidle_driver powernv_idle_driver = {
2c2e6ecf DD	32	.name = "powernv_idle",
	33	.owner = THIS_MODULE,
	34	};
	35
624e46d0 NP	36	static int max_idle_state __read_mostly;
624e46d0 NP	37	static struct cpuidle_state *cpuidle_state_table __read_mostly;
3005c597	38
09206b60 GS	39	struct stop_psscr_table {
	40	u64 val;
	41	u64 mask;
	42	};
	43
624e46d0	44	static struct stop_psscr_table stop_psscr_table[CPUIDLE_STATE_MAX] __read_mostly;
3005c597	45
0a4ec6aa	46	static u64 default_snooze_timeout __read_mostly;
624e46d0	47	static bool snooze_timeout_en __read_mostly;
2c2e6ecf	48
0a4ec6aa GS	49	static u64 get_snooze_timeout(struct cpuidle_device *dev,
	50	struct cpuidle_driver *drv,
	51	int index)
	52	{
	53	int i;
	54
	55	if (unlikely(!snooze_timeout_en))
	56	return default_snooze_timeout;
	57
	58	for (i = index + 1; i < drv->state_count; i++) {
	59	struct cpuidle_state *s = &drv->states[i];
	60	struct cpuidle_state_usage *su = &dev->states_usage[i];
	61
	62	if (s->disabled \|\| su->disable)
	63	continue;
	64
	65	return s->target_residency * tb_ticks_per_usec;
	66	}
	67
	68	return default_snooze_timeout;
	69	}
	70
2c2e6ecf DD	71	static int snooze_loop(struct cpuidle_device *dev,
	72	struct cpuidle_driver *drv,
	73	int index)
	74	{
78eaa10f SB	75	u64 snooze_exit_time;
78eaa10f SB	76
2c2e6ecf DD	77	set_thread_flag(TIF_POLLING_NRFLAG);
2c2e6ecf DD	78
3fc5ee92 NP	79	local_irq_enable();
3fc5ee92 NP	80
0a4ec6aa	81	snooze_exit_time = get_tb() + get_snooze_timeout(dev, drv, index);
591ac0cb	82	ppc64_runlatch_off();
26eb48a9	83	HMT_very_low();
2c2e6ecf	84	while (!need_resched()) {
7ded4291 NP	85	if (likely(snooze_timeout_en) && get_tb() > snooze_exit_time) {
	86	/*
	87	* Task has not woken up but we are exiting the polling
	88	* loop anyway. Require a barrier after polling is
	89	* cleared to order subsequent test of need_resched().
	90	*/
	91	clear_thread_flag(TIF_POLLING_NRFLAG);
	92	smp_mb();
78eaa10f	93	break;
7ded4291	94	}
2c2e6ecf DD	95	}
	96
	97	HMT_medium();
591ac0cb	98	ppc64_runlatch_on();
2c2e6ecf	99	clear_thread_flag(TIF_POLLING_NRFLAG);
3fc5ee92	100
f1343d04 NP	101	local_irq_disable();
f1343d04 NP	102
2c2e6ecf DD	103	return index;
	104	}
	105
	106	static int nap_loop(struct cpuidle_device *dev,
	107	struct cpuidle_driver *drv,
	108	int index)
	109	{
2201f994 NP	110	power7_idle_type(PNV_THREAD_NAP);
2201f994 NP	111
2c2e6ecf DD	112	return index;
	113	}
	114
cc5a2f7b	115	/* Register for fastsleep only in oneshot mode of broadcast */
cc5a2f7b	116	#ifdef CONFIG_TICK_ONESHOT
0d948730 PM	117	static int fastsleep_loop(struct cpuidle_device *dev,
	118	struct cpuidle_driver *drv,
	119	int index)
	120	{
	121	unsigned long old_lpcr = mfspr(SPRN_LPCR);
	122	unsigned long new_lpcr;
	123
	124	if (unlikely(system_state < SYSTEM_RUNNING))
	125	return index;
	126
	127	new_lpcr = old_lpcr;
9b6a68d9 MN	128	/* Do not exit powersave upon decrementer as we've setup the timer
9b6a68d9 MN	129	* offload.
0d948730	130	*/
9b6a68d9	131	new_lpcr &= ~LPCR_PECE1;
0d948730 PM	132
0d948730 PM	133	mtspr(SPRN_LPCR, new_lpcr);
2201f994 NP	134
2201f994 NP	135	power7_idle_type(PNV_THREAD_SLEEP);
0d948730 PM	136
	137	mtspr(SPRN_LPCR, old_lpcr);
	138
	139	return index;
	140	}
cc5a2f7b	141	#endif
3005c597 SP	142
	143	static int stop_loop(struct cpuidle_device *dev,
	144	struct cpuidle_driver *drv,
	145	int index)
	146	{
2201f994	147	power9_idle_type(stop_psscr_table[index].val,
09206b60	148	stop_psscr_table[index].mask);
3005c597 SP	149	return index;
	150	}
	151
2c2e6ecf DD	152	/*
	153	* States for dedicated partition case.
	154	*/
169f3fae	155	static struct cpuidle_state powernv_states[CPUIDLE_STATE_MAX] = {
2c2e6ecf DD	156	{ /* Snooze */
	157	.name = "snooze",
	158	.desc = "snooze",
2c2e6ecf DD	159	.exit_latency = 0,
2c2e6ecf DD	160	.target_residency = 0,
957efced	161	.enter = snooze_loop },
2c2e6ecf DD	162	};
2c2e6ecf DD	163
10fcca9d	164	static int powernv_cpuidle_cpu_online(unsigned int cpu)
2c2e6ecf	165	{
10fcca9d	166	struct cpuidle_device *dev = per_cpu(cpuidle_devices, cpu);
2c2e6ecf DD	167
2c2e6ecf DD	168	if (dev && cpuidle_get_driver()) {
10fcca9d SAS	169	cpuidle_pause_and_lock();
	170	cpuidle_enable_device(dev);
	171	cpuidle_resume_and_unlock();
	172	}
	173	return 0;
	174	}
2c2e6ecf	175
10fcca9d SAS	176	static int powernv_cpuidle_cpu_dead(unsigned int cpu)
	177	{
	178	struct cpuidle_device *dev = per_cpu(cpuidle_devices, cpu);
2c2e6ecf	179
10fcca9d SAS	180	if (dev && cpuidle_get_driver()) {
	181	cpuidle_pause_and_lock();
	182	cpuidle_disable_device(dev);
	183	cpuidle_resume_and_unlock();
2c2e6ecf	184	}
10fcca9d	185	return 0;
2c2e6ecf DD	186	}
2c2e6ecf DD	187
2c2e6ecf DD	188	/*
	189	* powernv_cpuidle_driver_init()
	190	*/
	191	static int powernv_cpuidle_driver_init(void)
	192	{
	193	int idle_state;
	194	struct cpuidle_driver *drv = &powernv_idle_driver;
	195
	196	drv->state_count = 0;
	197
	198	for (idle_state = 0; idle_state < max_idle_state; ++idle_state) {
	199	/* Is the state not enabled? */
	200	if (cpuidle_state_table[idle_state].enter == NULL)
	201	continue;
	202
	203	drv->states[drv->state_count] = /* structure copy */
	204	cpuidle_state_table[idle_state];
	205
	206	drv->state_count += 1;
	207	}
	208
293d264f VS	209	/*
	210	* On the PowerNV platform cpu_present may be less than cpu_possible in
	211	* cases when firmware detects the CPU, but it is not available to the
	212	* OS. If CONFIG_HOTPLUG_CPU=n, then such CPUs are not hotplugable at
	213	* run time and hence cpu_devices are not created for those CPUs by the
	214	* generic topology_init().
	215	*
	216	* drv->cpumask defaults to cpu_possible_mask in
	217	* __cpuidle_driver_init(). This breaks cpuidle on PowerNV where
	218	* cpu_devices are not created for CPUs in cpu_possible_mask that
	219	* cannot be hot-added later at run time.
	220	*
	221	* Trying cpuidle_register_device() on a CPU without a cpu_device is
	222	* incorrect, so pass a correct CPU mask to the generic cpuidle driver.
	223	*/
	224
	225	drv->cpumask = (struct cpumask *)cpu_present_mask;
	226
2c2e6ecf DD	227	return 0;
	228	}
	229
9e9fc6f0 GS	230	static inline void add_powernv_state(int index, const char *name,
	231	unsigned int flags,
	232	int (idle_fn)(struct cpuidle_device ,
	233	struct cpuidle_driver *,
	234	int),
	235	unsigned int target_residency,
	236	unsigned int exit_latency,
09206b60	237	u64 psscr_val, u64 psscr_mask)
9e9fc6f0 GS	238	{
	239	strlcpy(powernv_states[index].name, name, CPUIDLE_NAME_LEN);
	240	strlcpy(powernv_states[index].desc, name, CPUIDLE_NAME_LEN);
	241	powernv_states[index].flags = flags;
	242	powernv_states[index].target_residency = target_residency;
	243	powernv_states[index].exit_latency = exit_latency;
	244	powernv_states[index].enter = idle_fn;
1961acad	245	/* For power8 and below psscr_* will be 0 */
09206b60 GS	246	stop_psscr_table[index].val = psscr_val;
09206b60 GS	247	stop_psscr_table[index].mask = psscr_mask;
9e9fc6f0 GS	248	}
9e9fc6f0 GS	249
ecad4502 GS	250	/*
	251	* Returns 0 if prop1_len == prop2_len. Else returns -1
	252	*/
	253	static inline int validate_dt_prop_sizes(const char *prop1, int prop1_len,
	254	const char *prop2, int prop2_len)
	255	{
	256	if (prop1_len == prop2_len)
	257	return 0;
	258
	259	pr_warn("cpuidle-powernv: array sizes don't match for %s and %s\n",
	260	prop1, prop2);
	261	return -1;
	262	}
	263
785a12af	264	extern u32 pnv_get_supported_cpuidle_states(void);
0888839c PM	265	static int powernv_add_idle_states(void)
0888839c PM	266	{
0888839c	267	int nr_idle_states = 1; /* Snooze */
1961acad	268	int dt_idle_states;
09206b60	269	u32 has_stop_states = 0;
1961acad	270	int i;
785a12af GS	271	u32 supported_flags = pnv_get_supported_cpuidle_states();
785a12af GS	272
0888839c PM	273
0888839c PM	274	/* Currently we have snooze statically defined */
1961acad AA	275	if (nr_pnv_idle_states <= 0) {
1961acad AA	276	pr_warn("cpuidle-powernv : Only Snooze is available\n");
92c83ff5	277	goto out;
0888839c PM	278	}
0888839c PM	279
1961acad AA	280	/* TODO: Count only states which are eligible for cpuidle */
1961acad AA	281	dt_idle_states = nr_pnv_idle_states;
ecad4502	282
957efced SP	283	/*
	284	* Since snooze is used as first idle state, max idle states allowed is
	285	* CPUIDLE_STATE_MAX -1
	286	*/
1961acad	287	if (nr_pnv_idle_states > CPUIDLE_STATE_MAX - 1) {
957efced SP	288	pr_warn("cpuidle-powernv: discovered idle states more than allowed");
	289	dt_idle_states = CPUIDLE_STATE_MAX - 1;
	290	}
	291
3005c597 SP	292	/*
3005c597 SP	293	* If the idle states use stop instruction, probe for psscr values
09206b60	294	* and psscr mask which are necessary to specify required stop level.
3005c597	295	*/
1961acad	296	has_stop_states = (pnv_idle_states[0].flags &
09206b60	297	(OPAL_PM_STOP_INST_FAST \| OPAL_PM_STOP_INST_DEEP));
0888839c PM	298
0888839c PM	299	for (i = 0; i < dt_idle_states; i++) {
9e9fc6f0	300	unsigned int exit_latency, target_residency;
f9122ee4	301	bool stops_timebase = false;
1961acad	302	struct pnv_idle_states_t *state = &pnv_idle_states[i];
785a12af GS	303
	304	/*
	305	* Skip the platform idle state whose flag isn't in
	306	* the supported_cpuidle_states flag mask.
	307	*/
1961acad	308	if ((state->flags & supported_flags) != state->flags)
785a12af	309	continue;
3005c597 SP	310	/*
	311	* If an idle state has exit latency beyond
	312	* POWERNV_THRESHOLD_LATENCY_NS then don't use it
	313	* in cpu-idle.
	314	*/
1961acad	315	if (state->latency_ns > POWERNV_THRESHOLD_LATENCY_NS)
3005c597	316	continue;
9e9fc6f0 GS	317	/*
	318	* Firmware passes residency and latency values in ns.
	319	* cpuidle expects it in us.
	320	*/
1961acad AA	321	exit_latency = DIV_ROUND_UP(state->latency_ns, 1000);
	322	target_residency = DIV_ROUND_UP(state->residency_ns, 1000);
	323
	324	if (has_stop_states && !(state->valid))
09206b60	325	continue;
09206b60	326
1961acad	327	if (state->flags & OPAL_PM_TIMEBASE_STOP)
f9122ee4 GS	328	stops_timebase = true;
f9122ee4 GS	329
1961acad	330	if (state->flags & OPAL_PM_NAP_ENABLED) {
0888839c	331	/* Add NAP state */
9e9fc6f0 GS	332	add_powernv_state(nr_idle_states, "Nap",
9e9fc6f0 GS	333	CPUIDLE_FLAG_NONE, nap_loop,
09206b60	334	target_residency, exit_latency, 0, 0);
f9122ee4	335	} else if (has_stop_states && !stops_timebase) {
1961acad	336	add_powernv_state(nr_idle_states, state->name,
9e9fc6f0 GS	337	CPUIDLE_FLAG_NONE, stop_loop,
9e9fc6f0 GS	338	target_residency, exit_latency,
1961acad AA	339	state->psscr_val,
1961acad AA	340	state->psscr_mask);
cc5a2f7b	341	}
	342
	343	/*
	344	* All cpuidle states with CPUIDLE_FLAG_TIMER_STOP set must come
	345	* within this config dependency check.
	346	*/
	347	#ifdef CONFIG_TICK_ONESHOT
1961acad AA	348	else if (state->flags & OPAL_PM_SLEEP_ENABLED \|\|
1961acad AA	349	state->flags & OPAL_PM_SLEEP_ENABLED_ER1) {
0888839c	350	/* Add FASTSLEEP state */
9e9fc6f0 GS	351	add_powernv_state(nr_idle_states, "FastSleep",
	352	CPUIDLE_FLAG_TIMER_STOP,
	353	fastsleep_loop,
09206b60	354	target_residency, exit_latency, 0, 0);
f9122ee4	355	} else if (has_stop_states && stops_timebase) {
1961acad	356	add_powernv_state(nr_idle_states, state->name,
9e9fc6f0 GS	357	CPUIDLE_FLAG_TIMER_STOP, stop_loop,
9e9fc6f0 GS	358	target_residency, exit_latency,
1961acad AA	359	state->psscr_val,
1961acad AA	360	state->psscr_mask);
0888839c	361	}
cc5a2f7b	362	#endif
f9122ee4 GS	363	else
f9122ee4 GS	364	continue;
92c83ff5	365	nr_idle_states++;
0888839c	366	}
92c83ff5	367	out:
0888839c PM	368	return nr_idle_states;
	369	}
	370
2c2e6ecf DD	371	/*
	372	* powernv_idle_probe()
	373	* Choose state table for shared versus dedicated partition
	374	*/
	375	static int powernv_idle_probe(void)
	376	{
2c2e6ecf DD	377	if (cpuidle_disable != IDLE_NO_OVERRIDE)
	378	return -ENODEV;
	379
e4d54f71	380	if (firmware_has_feature(FW_FEATURE_OPAL)) {
2c2e6ecf	381	cpuidle_state_table = powernv_states;
0888839c PM	382	/* Device tree can indicate more idle states */
0888839c PM	383	max_idle_state = powernv_add_idle_states();
0a4ec6aa GS	384	default_snooze_timeout = TICK_USEC * tb_ticks_per_usec;
0a4ec6aa GS	385	if (max_idle_state > 1)
78eaa10f	386	snooze_timeout_en = true;
2c2e6ecf DD	387	} else
	388	return -ENODEV;
	389
	390	return 0;
	391	}
	392
	393	static int __init powernv_processor_idle_init(void)
	394	{
	395	int retval;
	396
	397	retval = powernv_idle_probe();
	398	if (retval)
	399	return retval;
	400
	401	powernv_cpuidle_driver_init();
	402	retval = cpuidle_register(&powernv_idle_driver, NULL);
	403	if (retval) {
	404	printk(KERN_DEBUG "Registration of powernv driver failed.\n");
	405	return retval;
	406	}
	407
10fcca9d SAS	408	retval = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
	409	"cpuidle/powernv:online",
	410	powernv_cpuidle_cpu_online, NULL);
	411	WARN_ON(retval < 0);
	412	retval = cpuhp_setup_state_nocalls(CPUHP_CPUIDLE_DEAD,
	413	"cpuidle/powernv:dead", NULL,
	414	powernv_cpuidle_cpu_dead);
	415	WARN_ON(retval < 0);
2c2e6ecf DD	416	printk(KERN_DEBUG "powernv_idle_driver registered\n");
	417	return 0;
	418	}
	419
	420	device_initcall(powernv_processor_idle_init);