[linux.git] / arch / x86 / xen / time.c

/*
 * Xen time implementation.
 *
 * This is implemented in terms of a clocksource driver which uses
 * the hypervisor clock as a nanosecond timebase, and a clockevent
 * driver which uses the hypervisor's timer mechanism.
 *
 * Jeremy Fitzhardinge <[email protected]>, XenSource Inc, 2007
 */
#include <linux/kernel.h>
#include <linux/interrupt.h>
#include <linux/clocksource.h>
#include <linux/clockchips.h>
#include <linux/kernel_stat.h>
#include <linux/math64.h>
#include <linux/gfp.h>
#include <linux/slab.h>
#include <linux/pvclock_gtod.h>

#include <asm/pvclock.h>
#include <asm/xen/hypervisor.h>
#include <asm/xen/hypercall.h>

#include <xen/events.h>
#include <xen/features.h>
#include <xen/interface/xen.h>
#include <xen/interface/vcpu.h>

#include "xen-ops.h"

/* Xen may fire a timer up to this many ns early */
#define TIMER_SLOP	100000
#define NS_PER_TICK	(1000000000LL / HZ)

/* runstate info updated by Xen */
static DEFINE_PER_CPU(struct vcpu_runstate_info, xen_runstate);

/* snapshots of runstate info */
static DEFINE_PER_CPU(struct vcpu_runstate_info, xen_runstate_snapshot);

/* unused ns of stolen time */
static DEFINE_PER_CPU(u64, xen_residual_stolen);

/* return an consistent snapshot of 64-bit time/counter value */
static u64 get64(const u64 *p)
{
	u64 ret;

	if (BITS_PER_LONG < 64) {
		u32 *p32 = (u32 *)p;
		u32 h, l;

		/*
		 * Read high then low, and then make sure high is
		 * still the same; this will only loop if low wraps
		 * and carries into high.
		 * XXX some clean way to make this endian-proof?
		 */
		do {
			h = p32[1];
			barrier();
			l = p32[0];
			barrier();
		} while (p32[1] != h);

		ret = (((u64)h) << 32) | l;
	} else
		ret = *p;

	return ret;
}

/*
 * Runstate accounting
 */
static void get_runstate_snapshot(struct vcpu_runstate_info *res)
{
	u64 state_time;
	struct vcpu_runstate_info *state;

	BUG_ON(preemptible());

	state = this_cpu_ptr(&xen_runstate);

	/*
	 * The runstate info is always updated by the hypervisor on
	 * the current CPU, so there's no need to use anything
	 * stronger than a compiler barrier when fetching it.
	 */
	do {
		state_time = get64(&state->state_entry_time);
		barrier();
		*res = *state;
		barrier();
	} while (get64(&state->state_entry_time) != state_time);
}

/* return true when a vcpu could run but has no real cpu to run on */
bool xen_vcpu_stolen(int vcpu)
{
	return per_cpu(xen_runstate, vcpu).state == RUNSTATE_runnable;
}

void xen_setup_runstate_info(int cpu)
{
	struct vcpu_register_runstate_memory_area area;

	area.addr.v = &per_cpu(xen_runstate, cpu);

	if (HYPERVISOR_vcpu_op(VCPUOP_register_runstate_memory_area,
			       cpu, &area))
		BUG();
}

static void do_stolen_accounting(void)
{
	struct vcpu_runstate_info state;
	struct vcpu_runstate_info *snap;
	s64 runnable, offline, stolen;
	cputime_t ticks;

	get_runstate_snapshot(&state);

	WARN_ON(state.state != RUNSTATE_running);

	snap = this_cpu_ptr(&xen_runstate_snapshot);

	/* work out how much time the VCPU has not been runn*ing*  */
	runnable = state.time[RUNSTATE_runnable] - snap->time[RUNSTATE_runnable];
	offline = state.time[RUNSTATE_offline] - snap->time[RUNSTATE_offline];

	*snap = state;

	/* Add the appropriate number of ticks of stolen time,
	   including any left-overs from last time. */
	stolen = runnable + offline + __this_cpu_read(xen_residual_stolen);

	if (stolen < 0)
		stolen = 0;

	ticks = iter_div_u64_rem(stolen, NS_PER_TICK, &stolen);
	__this_cpu_write(xen_residual_stolen, stolen);
	account_steal_ticks(ticks);
}

/* Get the TSC speed from Xen */
static unsigned long xen_tsc_khz(void)
{
	struct pvclock_vcpu_time_info *info =
		&HYPERVISOR_shared_info->vcpu_info[0].time;

	return pvclock_tsc_khz(info);
}

cycle_t xen_clocksource_read(void)
{
        struct pvclock_vcpu_time_info *src;
	cycle_t ret;

	preempt_disable_notrace();
	src = &__this_cpu_read(xen_vcpu)->time;
	ret = pvclock_clocksource_read(src);
	preempt_enable_notrace();
	return ret;
}

static cycle_t xen_clocksource_get_cycles(struct clocksource *cs)
{
	return xen_clocksource_read();
}

static void xen_read_wallclock(struct timespec *ts)
{
	struct shared_info *s = HYPERVISOR_shared_info;
	struct pvclock_wall_clock *wall_clock = &(s->wc);
        struct pvclock_vcpu_time_info *vcpu_time;

	vcpu_time = &get_cpu_var(xen_vcpu)->time;
	pvclock_read_wallclock(wall_clock, vcpu_time, ts);
	put_cpu_var(xen_vcpu);
}

static void xen_get_wallclock(struct timespec *now)
{
	xen_read_wallclock(now);
}

static int xen_set_wallclock(const struct timespec *now)
{
	return -1;
}

static int xen_pvclock_gtod_notify(struct notifier_block *nb,
				   unsigned long was_set, void *priv)
{
	/* Protected by the calling core code serialization */
	static struct timespec next_sync;

	struct xen_platform_op op;
	struct timespec now;

	now = __current_kernel_time();

	/*
	 * We only take the expensive HV call when the clock was set
	 * or when the 11 minutes RTC synchronization time elapsed.
	 */
	if (!was_set && timespec_compare(&now, &next_sync) < 0)
		return NOTIFY_OK;

	op.cmd = XENPF_settime;
	op.u.settime.secs = now.tv_sec;
	op.u.settime.nsecs = now.tv_nsec;
	op.u.settime.system_time = xen_clocksource_read();

	(void)HYPERVISOR_dom0_op(&op);

	/*
	 * Move the next drift compensation time 11 minutes
	 * ahead. That's emulating the sync_cmos_clock() update for
	 * the hardware RTC.
	 */
	next_sync = now;
	next_sync.tv_sec += 11 * 60;

	return NOTIFY_OK;
}

static struct notifier_block xen_pvclock_gtod_notifier = {
	.notifier_call = xen_pvclock_gtod_notify,
};

static struct clocksource xen_clocksource __read_mostly = {
	.name = "xen",
	.rating = 400,
	.read = xen_clocksource_get_cycles,
	.mask = ~0,
	.flags = CLOCK_SOURCE_IS_CONTINUOUS,
};

/*
   Xen clockevent implementation

   Xen has two clockevent implementations:

   The old timer_op one works with all released versions of Xen prior
   to version 3.0.4.  This version of the hypervisor provides a
   single-shot timer with nanosecond resolution.  However, sharing the
   same event channel is a 100Hz tick which is delivered while the
   vcpu is running.  We don't care about or use this tick, but it will
   cause the core time code to think the timer fired too soon, and
   will end up resetting it each time.  It could be filtered, but
   doing so has complications when the ktime clocksource is not yet
   the xen clocksource (ie, at boot time).

   The new vcpu_op-based timer interface allows the tick timer period
   to be changed or turned off.  The tick timer is not useful as a
   periodic timer because events are only delivered to running vcpus.
   The one-shot timer can report when a timeout is in the past, so
   set_next_event is capable of returning -ETIME when appropriate.
   This interface is used when available.
*/


/*
  Get a hypervisor absolute time.  In theory we could maintain an
  offset between the kernel's time and the hypervisor's time, and
  apply that to a kernel's absolute timeout.  Unfortunately the
  hypervisor and kernel times can drift even if the kernel is using
  the Xen clocksource, because ntp can warp the kernel's clocksource.
*/
static s64 get_abs_timeout(unsigned long delta)
{
	return xen_clocksource_read() + delta;
}

static int xen_timerop_shutdown(struct clock_event_device *evt)
{
	/* cancel timeout */
	HYPERVISOR_set_timer_op(0);

	return 0;
}

static int xen_timerop_set_next_event(unsigned long delta,
				      struct clock_event_device *evt)
{
	WARN_ON(!clockevent_state_oneshot(evt));

	if (HYPERVISOR_set_timer_op(get_abs_timeout(delta)) < 0)
		BUG();

	/* We may have missed the deadline, but there's no real way of
	   knowing for sure.  If the event was in the past, then we'll
	   get an immediate interrupt. */

	return 0;
}

static const struct clock_event_device xen_timerop_clockevent = {
	.name			= "xen",
	.features		= CLOCK_EVT_FEAT_ONESHOT,

	.max_delta_ns		= 0xffffffff,
	.min_delta_ns		= TIMER_SLOP,

	.mult			= 1,
	.shift			= 0,
	.rating			= 500,

	.set_state_shutdown	= xen_timerop_shutdown,
	.set_next_event		= xen_timerop_set_next_event,
};

static int xen_vcpuop_shutdown(struct clock_event_device *evt)
{
	int cpu = smp_processor_id();

	if (HYPERVISOR_vcpu_op(VCPUOP_stop_singleshot_timer, cpu, NULL) ||
	    HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL))
		BUG();

	return 0;
}

static int xen_vcpuop_set_oneshot(struct clock_event_device *evt)
{
	int cpu = smp_processor_id();

	if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL))
		BUG();

	return 0;
}

static int xen_vcpuop_set_next_event(unsigned long delta,
				     struct clock_event_device *evt)
{
	int cpu = smp_processor_id();
	struct vcpu_set_singleshot_timer single;
	int ret;

	WARN_ON(!clockevent_state_oneshot(evt));

	single.timeout_abs_ns = get_abs_timeout(delta);
	single.flags = VCPU_SSHOTTMR_future;

	ret = HYPERVISOR_vcpu_op(VCPUOP_set_singleshot_timer, cpu, &single);

	BUG_ON(ret != 0 && ret != -ETIME);

	return ret;
}

static const struct clock_event_device xen_vcpuop_clockevent = {
	.name = "xen",
	.features = CLOCK_EVT_FEAT_ONESHOT,

	.max_delta_ns = 0xffffffff,
	.min_delta_ns = TIMER_SLOP,

	.mult = 1,
	.shift = 0,
	.rating = 500,

	.set_state_shutdown = xen_vcpuop_shutdown,
	.set_state_oneshot = xen_vcpuop_set_oneshot,
	.set_next_event = xen_vcpuop_set_next_event,
};

static const struct clock_event_device *xen_clockevent =
	&xen_timerop_clockevent;

struct xen_clock_event_device {
	struct clock_event_device evt;
	char name[16];
};
static DEFINE_PER_CPU(struct xen_clock_event_device, xen_clock_events) = { .evt.irq = -1 };

static irqreturn_t xen_timer_interrupt(int irq, void *dev_id)
{
	struct clock_event_device *evt = this_cpu_ptr(&xen_clock_events.evt);
	irqreturn_t ret;

	ret = IRQ_NONE;
	if (evt->event_handler) {
		evt->event_handler(evt);
		ret = IRQ_HANDLED;
	}

	do_stolen_accounting();

	return ret;
}

void xen_teardown_timer(int cpu)
{
	struct clock_event_device *evt;
	BUG_ON(cpu == 0);
	evt = &per_cpu(xen_clock_events, cpu).evt;

	if (evt->irq >= 0) {
		unbind_from_irqhandler(evt->irq, NULL);
		evt->irq = -1;
	}
}

void xen_setup_timer(int cpu)
{
	struct xen_clock_event_device *xevt = &per_cpu(xen_clock_events, cpu);
	struct clock_event_device *evt = &xevt->evt;
	int irq;

	WARN(evt->irq >= 0, "IRQ%d for CPU%d is already allocated\n", evt->irq, cpu);
	if (evt->irq >= 0)
		xen_teardown_timer(cpu);

	printk(KERN_INFO "installing Xen timer for CPU %d\n", cpu);

	snprintf(xevt->name, sizeof(xevt->name), "timer%d", cpu);

	irq = bind_virq_to_irqhandler(VIRQ_TIMER, cpu, xen_timer_interrupt,
				      IRQF_PERCPU|IRQF_NOBALANCING|IRQF_TIMER|
				      IRQF_FORCE_RESUME|IRQF_EARLY_RESUME,
				      xevt->name, NULL);
	(void)xen_set_irq_priority(irq, XEN_IRQ_PRIORITY_MAX);

	memcpy(evt, xen_clockevent, sizeof(*evt));

	evt->cpumask = cpumask_of(cpu);
	evt->irq = irq;
}


void xen_setup_cpu_clockevents(void)
{
	clockevents_register_device(this_cpu_ptr(&xen_clock_events.evt));
}

void xen_timer_resume(void)
{
	int cpu;

	pvclock_resume();

	if (xen_clockevent != &xen_vcpuop_clockevent)
		return;

	for_each_online_cpu(cpu) {
		if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL))
			BUG();
	}
}

static const struct pv_time_ops xen_time_ops __initconst = {
	.sched_clock = xen_clocksource_read,
};

static void __init xen_time_init(void)
{
	int cpu = smp_processor_id();
	struct timespec tp;

	/* As Dom0 is never moved, no penalty on using TSC there */
	if (xen_initial_domain())
		xen_clocksource.rating = 275;

	clocksource_register_hz(&xen_clocksource, NSEC_PER_SEC);

	if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL) == 0) {
		/* Successfully turned off 100Hz tick, so we have the
		   vcpuop-based timer interface */
		printk(KERN_DEBUG "Xen: using vcpuop timer interface\n");
		xen_clockevent = &xen_vcpuop_clockevent;
	}

	/* Set initial system time with full resolution */
	xen_read_wallclock(&tp);
	do_settimeofday(&tp);

	setup_force_cpu_cap(X86_FEATURE_TSC);

	xen_setup_runstate_info(cpu);
	xen_setup_timer(cpu);
	xen_setup_cpu_clockevents();

	if (xen_initial_domain())
		pvclock_gtod_register_notifier(&xen_pvclock_gtod_notifier);
}

void __init xen_init_time_ops(void)
{
	pv_time_ops = xen_time_ops;

	x86_init.timers.timer_init = xen_time_init;
	x86_init.timers.setup_percpu_clockev = x86_init_noop;
	x86_cpuinit.setup_percpu_clockev = x86_init_noop;

	x86_platform.calibrate_tsc = xen_tsc_khz;
	x86_platform.get_wallclock = xen_get_wallclock;
	/* Dom0 uses the native method to set the hardware RTC. */
	if (!xen_initial_domain())
		x86_platform.set_wallclock = xen_set_wallclock;
}

#ifdef CONFIG_XEN_PVHVM
static void xen_hvm_setup_cpu_clockevents(void)
{
	int cpu = smp_processor_id();
	xen_setup_runstate_info(cpu);
	/*
	 * xen_setup_timer(cpu) - snprintf is bad in atomic context. Hence
	 * doing it xen_hvm_cpu_notify (which gets called by smp_init during
	 * early bootup and also during CPU hotplug events).
	 */
	xen_setup_cpu_clockevents();
}

void __init xen_hvm_init_time_ops(void)
{
	/* vector callback is needed otherwise we cannot receive interrupts
	 * on cpu > 0 and at this point we don't know how many cpus are
	 * available */
	if (!xen_have_vector_callback)
		return;
	if (!xen_feature(XENFEAT_hvm_safe_pvclock)) {
		printk(KERN_INFO "Xen doesn't support pvclock on HVM,"
				"disable pv timer\n");
		return;
	}

	pv_time_ops = xen_time_ops;
	x86_init.timers.setup_percpu_clockev = xen_time_init;
	x86_cpuinit.setup_percpu_clockev = xen_hvm_setup_cpu_clockevents;

	x86_platform.calibrate_tsc = xen_tsc_khz;
	x86_platform.get_wallclock = xen_get_wallclock;
	x86_platform.set_wallclock = xen_set_wallclock;
}
#endif
Commit	Line	Data
15c84731 JF	1	/*
	2	* Xen time implementation.
	3	*
	4	* This is implemented in terms of a clocksource driver which uses
	5	* the hypervisor clock as a nanosecond timebase, and a clockevent
	6	* driver which uses the hypervisor's timer mechanism.
	7	*
	8	* Jeremy Fitzhardinge <[email protected]>, XenSource Inc, 2007
	9	*/
	10	#include <linux/kernel.h>
	11	#include <linux/interrupt.h>
	12	#include <linux/clocksource.h>
	13	#include <linux/clockchips.h>
f91a8b44	14	#include <linux/kernel_stat.h>
f595ec96	15	#include <linux/math64.h>
5a0e3ad6	16	#include <linux/gfp.h>
c9d76a24	17	#include <linux/slab.h>
5584880e	18	#include <linux/pvclock_gtod.h>
15c84731	19
1c7b67f7	20	#include <asm/pvclock.h>
15c84731 JF	21	#include <asm/xen/hypervisor.h>
	22	#include <asm/xen/hypercall.h>
	23
	24	#include <xen/events.h>
409771d2	25	#include <xen/features.h>
15c84731 JF	26	#include <xen/interface/xen.h>
	27	#include <xen/interface/vcpu.h>
	28
	29	#include "xen-ops.h"
	30
15c84731 JF	31	/* Xen may fire a timer up to this many ns early */
15c84731 JF	32	#define TIMER_SLOP 100000
f91a8b44	33	#define NS_PER_TICK (1000000000LL / HZ)
15c84731	34
f91a8b44	35	/* runstate info updated by Xen */
c6e22f9e	36	static DEFINE_PER_CPU(struct vcpu_runstate_info, xen_runstate);
f91a8b44 JF	37
f91a8b44 JF	38	/* snapshots of runstate info */
c6e22f9e	39	static DEFINE_PER_CPU(struct vcpu_runstate_info, xen_runstate_snapshot);
f91a8b44	40
0b0c002c	41	/* unused ns of stolen time */
c6e22f9e	42	static DEFINE_PER_CPU(u64, xen_residual_stolen);
f91a8b44 JF	43
	44	/* return an consistent snapshot of 64-bit time/counter value */
	45	static u64 get64(const u64 *p)
	46	{
	47	u64 ret;
	48
	49	if (BITS_PER_LONG < 64) {
	50	u32 p32 = (u32 )p;
	51	u32 h, l;
	52
	53	/*
	54	* Read high then low, and then make sure high is
	55	* still the same; this will only loop if low wraps
	56	* and carries into high.
	57	* XXX some clean way to make this endian-proof?
	58	*/
	59	do {
	60	h = p32[1];
	61	barrier();
	62	l = p32[0];
	63	barrier();
	64	} while (p32[1] != h);
	65
	66	ret = (((u64)h) << 32) \| l;
	67	} else
	68	ret = *p;
	69
	70	return ret;
	71	}
	72
	73	/*
	74	* Runstate accounting
	75	*/
	76	static void get_runstate_snapshot(struct vcpu_runstate_info *res)
	77	{
	78	u64 state_time;
	79	struct vcpu_runstate_info *state;
	80
f120f13e	81	BUG_ON(preemptible());
f91a8b44	82
89cbc767	83	state = this_cpu_ptr(&xen_runstate);
f91a8b44 JF	84
	85	/*
	86	* The runstate info is always updated by the hypervisor on
	87	* the current CPU, so there's no need to use anything
	88	* stronger than a compiler barrier when fetching it.
	89	*/
	90	do {
	91	state_time = get64(&state->state_entry_time);
	92	barrier();
	93	res = state;
	94	barrier();
	95	} while (get64(&state->state_entry_time) != state_time);
f91a8b44 JF	96	}
f91a8b44 JF	97
f0d73394 JF	98	/* return true when a vcpu could run but has no real cpu to run on */
	99	bool xen_vcpu_stolen(int vcpu)
	100	{
c6e22f9e	101	return per_cpu(xen_runstate, vcpu).state == RUNSTATE_runnable;
f0d73394 JF	102	}
f0d73394 JF	103
be012920	104	void xen_setup_runstate_info(int cpu)
f91a8b44 JF	105	{
	106	struct vcpu_register_runstate_memory_area area;
	107
c6e22f9e	108	area.addr.v = &per_cpu(xen_runstate, cpu);
f91a8b44 JF	109
	110	if (HYPERVISOR_vcpu_op(VCPUOP_register_runstate_memory_area,
	111	cpu, &area))
	112	BUG();
	113	}
	114
	115	static void do_stolen_accounting(void)
	116	{
	117	struct vcpu_runstate_info state;
	118	struct vcpu_runstate_info *snap;
0b0c002c	119	s64 runnable, offline, stolen;
f91a8b44 JF	120	cputime_t ticks;
	121
	122	get_runstate_snapshot(&state);
	123
	124	WARN_ON(state.state != RUNSTATE_running);
	125
89cbc767	126	snap = this_cpu_ptr(&xen_runstate_snapshot);
f91a8b44 JF	127
f91a8b44 JF	128	/* work out how much time the VCPU has not been running */
f91a8b44 JF	129	runnable = state.time[RUNSTATE_runnable] - snap->time[RUNSTATE_runnable];
	130	offline = state.time[RUNSTATE_offline] - snap->time[RUNSTATE_offline];
	131
	132	*snap = state;
	133
	134	/* Add the appropriate number of ticks of stolen time,
79741dd3	135	including any left-overs from last time. */
780f36d8	136	stolen = runnable + offline + __this_cpu_read(xen_residual_stolen);
f91a8b44 JF	137
	138	if (stolen < 0)
	139	stolen = 0;
	140
f595ec96	141	ticks = iter_div_u64_rem(stolen, NS_PER_TICK, &stolen);
780f36d8	142	__this_cpu_write(xen_residual_stolen, stolen);
79741dd3	143	account_steal_ticks(ticks);
f91a8b44 JF	144	}
f91a8b44 JF	145
e93ef949	146	/* Get the TSC speed from Xen */
409771d2	147	static unsigned long xen_tsc_khz(void)
15c84731	148	{
3807f345	149	struct pvclock_vcpu_time_info *info =
15c84731 JF	150	&HYPERVISOR_shared_info->vcpu_info[0].time;
15c84731 JF	151
3807f345	152	return pvclock_tsc_khz(info);
15c84731 JF	153	}
15c84731 JF	154
ee7686bc	155	cycle_t xen_clocksource_read(void)
15c84731	156	{
1c7b67f7	157	struct pvclock_vcpu_time_info *src;
15c84731	158	cycle_t ret;
15c84731	159
f1c39625	160	preempt_disable_notrace();
3251f20b	161	src = &__this_cpu_read(xen_vcpu)->time;
1c7b67f7	162	ret = pvclock_clocksource_read(src);
f1c39625	163	preempt_enable_notrace();
15c84731 JF	164	return ret;
	165	}
	166
8e19608e MD	167	static cycle_t xen_clocksource_get_cycles(struct clocksource *cs)
	168	{
	169	return xen_clocksource_read();
	170	}
	171
15c84731 JF	172	static void xen_read_wallclock(struct timespec *ts)
15c84731 JF	173	{
1c7b67f7 GH	174	struct shared_info *s = HYPERVISOR_shared_info;
	175	struct pvclock_wall_clock *wall_clock = &(s->wc);
	176	struct pvclock_vcpu_time_info *vcpu_time;
15c84731	177
1c7b67f7 GH	178	vcpu_time = &get_cpu_var(xen_vcpu)->time;
	179	pvclock_read_wallclock(wall_clock, vcpu_time, ts);
	180	put_cpu_var(xen_vcpu);
15c84731 JF	181	}
15c84731 JF	182
3565184e	183	static void xen_get_wallclock(struct timespec *now)
15c84731	184	{
3565184e	185	xen_read_wallclock(now);
15c84731	186	}
15c84731	187
3565184e	188	static int xen_set_wallclock(const struct timespec *now)
15c84731	189	{
47433b8c	190	return -1;
15c84731 JF	191	}
15c84731 JF	192
47433b8c DV	193	static int xen_pvclock_gtod_notify(struct notifier_block *nb,
47433b8c DV	194	unsigned long was_set, void *priv)
15c84731	195	{
47433b8c DV	196	/* Protected by the calling core code serialization */
47433b8c DV	197	static struct timespec next_sync;
5584880e	198
fdb9eb9f	199	struct xen_platform_op op;
47433b8c	200	struct timespec now;
fdb9eb9f	201
5584880e DV	202	now = __current_kernel_time();
5584880e DV	203
47433b8c DV	204	/*
	205	* We only take the expensive HV call when the clock was set
	206	* or when the 11 minutes RTC synchronization time elapsed.
	207	*/
	208	if (!was_set && timespec_compare(&now, &next_sync) < 0)
	209	return NOTIFY_OK;
fdb9eb9f JF	210
fdb9eb9f JF	211	op.cmd = XENPF_settime;
5584880e DV	212	op.u.settime.secs = now.tv_sec;
5584880e DV	213	op.u.settime.nsecs = now.tv_nsec;
fdb9eb9f JF	214	op.u.settime.system_time = xen_clocksource_read();
fdb9eb9f JF	215
5584880e	216	(void)HYPERVISOR_dom0_op(&op);
fdb9eb9f	217
47433b8c DV	218	/*
	219	* Move the next drift compensation time 11 minutes
	220	* ahead. That's emulating the sync_cmos_clock() update for
	221	* the hardware RTC.
	222	*/
	223	next_sync = now;
	224	next_sync.tv_sec += 11 * 60;
	225
5584880e	226	return NOTIFY_OK;
15c84731 JF	227	}
15c84731 JF	228
5584880e DV	229	static struct notifier_block xen_pvclock_gtod_notifier = {
	230	.notifier_call = xen_pvclock_gtod_notify,
	231	};
	232
15c84731 JF	233	static struct clocksource xen_clocksource __read_mostly = {
	234	.name = "xen",
	235	.rating = 400,
8e19608e	236	.read = xen_clocksource_get_cycles,
15c84731	237	.mask = ~0,
15c84731 JF	238	.flags = CLOCK_SOURCE_IS_CONTINUOUS,
	239	};
	240
	241	/*
	242	Xen clockevent implementation
	243
	244	Xen has two clockevent implementations:
	245
	246	The old timer_op one works with all released versions of Xen prior
	247	to version 3.0.4. This version of the hypervisor provides a
	248	single-shot timer with nanosecond resolution. However, sharing the
	249	same event channel is a 100Hz tick which is delivered while the
	250	vcpu is running. We don't care about or use this tick, but it will
	251	cause the core time code to think the timer fired too soon, and
	252	will end up resetting it each time. It could be filtered, but
	253	doing so has complications when the ktime clocksource is not yet
	254	the xen clocksource (ie, at boot time).
	255
	256	The new vcpu_op-based timer interface allows the tick timer period
	257	to be changed or turned off. The tick timer is not useful as a
	258	periodic timer because events are only delivered to running vcpus.
	259	The one-shot timer can report when a timeout is in the past, so
	260	set_next_event is capable of returning -ETIME when appropriate.
	261	This interface is used when available.
	262	*/
	263
	264
	265	/*
	266	Get a hypervisor absolute time. In theory we could maintain an
	267	offset between the kernel's time and the hypervisor's time, and
	268	apply that to a kernel's absolute timeout. Unfortunately the
	269	hypervisor and kernel times can drift even if the kernel is using
	270	the Xen clocksource, because ntp can warp the kernel's clocksource.
	271	*/
	272	static s64 get_abs_timeout(unsigned long delta)
	273	{
	274	return xen_clocksource_read() + delta;
	275	}
	276
955381dd	277	static int xen_timerop_shutdown(struct clock_event_device *evt)
15c84731	278	{
955381dd VK	279	/* cancel timeout */
	280	HYPERVISOR_set_timer_op(0);
	281
	282	return 0;
15c84731 JF	283	}
	284
	285	static int xen_timerop_set_next_event(unsigned long delta,
	286	struct clock_event_device *evt)
	287	{
955381dd	288	WARN_ON(!clockevent_state_oneshot(evt));
15c84731 JF	289
	290	if (HYPERVISOR_set_timer_op(get_abs_timeout(delta)) < 0)
	291	BUG();
	292
	293	/* We may have missed the deadline, but there's no real way of
	294	knowing for sure. If the event was in the past, then we'll
	295	get an immediate interrupt. */
	296
	297	return 0;
	298	}
	299
	300	static const struct clock_event_device xen_timerop_clockevent = {
955381dd VK	301	.name = "xen",
955381dd VK	302	.features = CLOCK_EVT_FEAT_ONESHOT,
15c84731	303
955381dd VK	304	.max_delta_ns = 0xffffffff,
955381dd VK	305	.min_delta_ns = TIMER_SLOP,
15c84731	306
955381dd VK	307	.mult = 1,
	308	.shift = 0,
	309	.rating = 500,
15c84731	310
955381dd VK	311	.set_state_shutdown = xen_timerop_shutdown,
955381dd VK	312	.set_next_event = xen_timerop_set_next_event,
15c84731 JF	313	};
15c84731 JF	314
955381dd VK	315	static int xen_vcpuop_shutdown(struct clock_event_device *evt)
	316	{
	317	int cpu = smp_processor_id();
15c84731	318
955381dd VK	319	if (HYPERVISOR_vcpu_op(VCPUOP_stop_singleshot_timer, cpu, NULL) \|\|
	320	HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL))
	321	BUG();
15c84731	322
955381dd VK	323	return 0;
	324	}
	325
	326	static int xen_vcpuop_set_oneshot(struct clock_event_device *evt)
15c84731 JF	327	{
	328	int cpu = smp_processor_id();
	329
955381dd VK	330	if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL))
	331	BUG();
	332
	333	return 0;
15c84731 JF	334	}
	335
	336	static int xen_vcpuop_set_next_event(unsigned long delta,
	337	struct clock_event_device *evt)
	338	{
	339	int cpu = smp_processor_id();
	340	struct vcpu_set_singleshot_timer single;
	341	int ret;
	342
955381dd	343	WARN_ON(!clockevent_state_oneshot(evt));
15c84731 JF	344
	345	single.timeout_abs_ns = get_abs_timeout(delta);
	346	single.flags = VCPU_SSHOTTMR_future;
	347
	348	ret = HYPERVISOR_vcpu_op(VCPUOP_set_singleshot_timer, cpu, &single);
	349
	350	BUG_ON(ret != 0 && ret != -ETIME);
	351
	352	return ret;
	353	}
	354
	355	static const struct clock_event_device xen_vcpuop_clockevent = {
	356	.name = "xen",
	357	.features = CLOCK_EVT_FEAT_ONESHOT,
	358
	359	.max_delta_ns = 0xffffffff,
	360	.min_delta_ns = TIMER_SLOP,
	361
	362	.mult = 1,
	363	.shift = 0,
	364	.rating = 500,
	365
955381dd VK	366	.set_state_shutdown = xen_vcpuop_shutdown,
955381dd VK	367	.set_state_oneshot = xen_vcpuop_set_oneshot,
15c84731 JF	368	.set_next_event = xen_vcpuop_set_next_event,
	369	};
	370
	371	static const struct clock_event_device *xen_clockevent =
	372	&xen_timerop_clockevent;
31620a19 KRW	373
	374	struct xen_clock_event_device {
	375	struct clock_event_device evt;
7be0772d	376	char name[16];
31620a19 KRW	377	};
31620a19 KRW	378	static DEFINE_PER_CPU(struct xen_clock_event_device, xen_clock_events) = { .evt.irq = -1 };
15c84731 JF	379
	380	static irqreturn_t xen_timer_interrupt(int irq, void *dev_id)
	381	{
89cbc767	382	struct clock_event_device *evt = this_cpu_ptr(&xen_clock_events.evt);
15c84731 JF	383	irqreturn_t ret;
	384
	385	ret = IRQ_NONE;
	386	if (evt->event_handler) {
	387	evt->event_handler(evt);
	388	ret = IRQ_HANDLED;
	389	}
	390
f91a8b44 JF	391	do_stolen_accounting();
f91a8b44 JF	392
15c84731 JF	393	return ret;
	394	}
	395
09e99da7 KRW	396	void xen_teardown_timer(int cpu)
	397	{
	398	struct clock_event_device *evt;
	399	BUG_ON(cpu == 0);
	400	evt = &per_cpu(xen_clock_events, cpu).evt;
	401
	402	if (evt->irq >= 0) {
	403	unbind_from_irqhandler(evt->irq, NULL);
	404	evt->irq = -1;
09e99da7 KRW	405	}
	406	}
	407
f87e4cac	408	void xen_setup_timer(int cpu)
15c84731	409	{
7be0772d VK	410	struct xen_clock_event_device *xevt = &per_cpu(xen_clock_events, cpu);
7be0772d VK	411	struct clock_event_device *evt = &xevt->evt;
15c84731 JF	412	int irq;
15c84731 JF	413
ef35a4e6	414	WARN(evt->irq >= 0, "IRQ%d for CPU%d is already allocated\n", evt->irq, cpu);
09e99da7 KRW	415	if (evt->irq >= 0)
09e99da7 KRW	416	xen_teardown_timer(cpu);
ef35a4e6	417
15c84731 JF	418	printk(KERN_INFO "installing Xen timer for CPU %d\n", cpu);
15c84731 JF	419
7be0772d	420	snprintf(xevt->name, sizeof(xevt->name), "timer%d", cpu);
15c84731 JF	421
15c84731 JF	422	irq = bind_virq_to_irqhandler(VIRQ_TIMER, cpu, xen_timer_interrupt,
9d71cee6	423	IRQF_PERCPU\|IRQF_NOBALANCING\|IRQF_TIMER\|
8d5999df	424	IRQF_FORCE_RESUME\|IRQF_EARLY_RESUME,
7be0772d	425	xevt->name, NULL);
8785c676	426	(void)xen_set_irq_priority(irq, XEN_IRQ_PRIORITY_MAX);
15c84731	427
15c84731 JF	428	memcpy(evt, xen_clockevent, sizeof(*evt));
15c84731 JF	429
320ab2b0	430	evt->cpumask = cpumask_of(cpu);
15c84731	431	evt->irq = irq;
f87e4cac JF	432	}
f87e4cac JF	433
d68d82af	434
f87e4cac JF	435	void xen_setup_cpu_clockevents(void)
f87e4cac JF	436	{
89cbc767	437	clockevents_register_device(this_cpu_ptr(&xen_clock_events.evt));
15c84731 JF	438	}
15c84731 JF	439
d07af1f0 JF	440	void xen_timer_resume(void)
	441	{
	442	int cpu;
	443
e7a3481c JF	444	pvclock_resume();
e7a3481c JF	445
d07af1f0 JF	446	if (xen_clockevent != &xen_vcpuop_clockevent)
	447	return;
	448
	449	for_each_online_cpu(cpu) {
	450	if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL))
	451	BUG();
	452	}
	453	}
	454
fb6ce5de	455	static const struct pv_time_ops xen_time_ops __initconst = {
ca50a5f3	456	.sched_clock = xen_clocksource_read,
409771d2 SS	457	};
409771d2 SS	458
fb6ce5de	459	static void __init xen_time_init(void)
15c84731 JF	460	{
15c84731 JF	461	int cpu = smp_processor_id();
c4507257	462	struct timespec tp;
15c84731	463
94dd85f6 PI	464	/* As Dom0 is never moved, no penalty on using TSC there */
	465	if (xen_initial_domain())
	466	xen_clocksource.rating = 275;
	467
b01cc1b0	468	clocksource_register_hz(&xen_clocksource, NSEC_PER_SEC);
15c84731 JF	469
15c84731 JF	470	if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL) == 0) {
f91a8b44	471	/* Successfully turned off 100Hz tick, so we have the
15c84731 JF	472	vcpuop-based timer interface */
	473	printk(KERN_DEBUG "Xen: using vcpuop timer interface\n");
	474	xen_clockevent = &xen_vcpuop_clockevent;
	475	}
	476
	477	/* Set initial system time with full resolution */
c4507257 JS	478	xen_read_wallclock(&tp);
c4507257 JS	479	do_settimeofday(&tp);
15c84731	480
404ee5b1	481	setup_force_cpu_cap(X86_FEATURE_TSC);
15c84731	482
be012920	483	xen_setup_runstate_info(cpu);
15c84731	484	xen_setup_timer(cpu);
f87e4cac	485	xen_setup_cpu_clockevents();
5584880e DV	486
	487	if (xen_initial_domain())
	488	pvclock_gtod_register_notifier(&xen_pvclock_gtod_notifier);
15c84731	489	}
409771d2	490
fb6ce5de	491	void __init xen_init_time_ops(void)
409771d2 SS	492	{
	493	pv_time_ops = xen_time_ops;
	494
	495	x86_init.timers.timer_init = xen_time_init;
	496	x86_init.timers.setup_percpu_clockev = x86_init_noop;
	497	x86_cpuinit.setup_percpu_clockev = x86_init_noop;
	498
	499	x86_platform.calibrate_tsc = xen_tsc_khz;
	500	x86_platform.get_wallclock = xen_get_wallclock;
47433b8c DV	501	/* Dom0 uses the native method to set the hardware RTC. */
	502	if (!xen_initial_domain())
	503	x86_platform.set_wallclock = xen_set_wallclock;
409771d2 SS	504	}
409771d2 SS	505
ca65f9fc	506	#ifdef CONFIG_XEN_PVHVM
409771d2 SS	507	static void xen_hvm_setup_cpu_clockevents(void)
	508	{
	509	int cpu = smp_processor_id();
	510	xen_setup_runstate_info(cpu);
7918c92a KRW	511	/*
	512	* xen_setup_timer(cpu) - snprintf is bad in atomic context. Hence
	513	* doing it xen_hvm_cpu_notify (which gets called by smp_init during
	514	* early bootup and also during CPU hotplug events).
	515	*/
409771d2 SS	516	xen_setup_cpu_clockevents();
	517	}
	518
fb6ce5de	519	void __init xen_hvm_init_time_ops(void)
409771d2 SS	520	{
409771d2 SS	521	/* vector callback is needed otherwise we cannot receive interrupts
31e7e931 SS	522	* on cpu > 0 and at this point we don't know how many cpus are
	523	* available */
	524	if (!xen_have_vector_callback)
409771d2 SS	525	return;
	526	if (!xen_feature(XENFEAT_hvm_safe_pvclock)) {
	527	printk(KERN_INFO "Xen doesn't support pvclock on HVM,"
	528	"disable pv timer\n");
	529	return;
	530	}
	531
	532	pv_time_ops = xen_time_ops;
	533	x86_init.timers.setup_percpu_clockev = xen_time_init;
	534	x86_cpuinit.setup_percpu_clockev = xen_hvm_setup_cpu_clockevents;
	535
	536	x86_platform.calibrate_tsc = xen_tsc_khz;
	537	x86_platform.get_wallclock = xen_get_wallclock;
	538	x86_platform.set_wallclock = xen_set_wallclock;
	539	}
ca65f9fc	540	#endif