From ad2b13536ace08dfcca4cf86b75a5d06efe06373 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 5 Mar 2013 15:14:05 +0100 Subject: tick: Call tick_init late To convert the clockevents code to cpumask_var_t we need to move the init call after the allocator setup. Clockevents are earliest registered from time_init() as they need interrupts being set up, so this is safe. Signed-off-by: Thomas Gleixner Link: http://lkml.kernel.org/r/20130306111537.304379448@linutronix.de Cc: Rusty Russell --- init/main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init/main.c b/init/main.c index 63534a141b4e..b3e061428545 100644 --- a/init/main.c +++ b/init/main.c @@ -494,7 +494,6 @@ asmlinkage void __init start_kernel(void) * Interrupts are still disabled. Do necessary setups, then * enable them */ - tick_init(); boot_cpu_init(); page_address_init(); printk(KERN_NOTICE "%s", linux_banner); @@ -551,6 +550,7 @@ asmlinkage void __init start_kernel(void) /* init some links before init_ISA_irqs() */ early_irq_init(); init_IRQ(); + tick_init(); init_timers(); hrtimers_init(); softirq_init(); -- cgit v1.2.3 From b352bc1cbc29134a356b5c16ee2281807a7b984e Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Tue, 5 Mar 2013 14:25:32 +0100 Subject: tick: Convert broadcast cpu bitmaps to cpumask_var_t Signed-off-by: Thomas Gleixner Link: http://lkml.kernel.org/r/20130306111537.366394000@linutronix.de Cc: Rusty Russell --- kernel/time/tick-broadcast.c | 86 ++++++++++++++++++++++---------------------- kernel/time/tick-common.c | 1 + kernel/time/tick-internal.h | 3 +- 3 files changed, 46 insertions(+), 44 deletions(-) diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c index 2fb8cb88df8d..35b887517766 100644 --- a/kernel/time/tick-broadcast.c +++ b/kernel/time/tick-broadcast.c @@ -28,9 +28,8 @@ */ static struct tick_device tick_broadcast_device; -/* FIXME: Use cpumask_var_t. */ -static DECLARE_BITMAP(tick_broadcast_mask, NR_CPUS); -static DECLARE_BITMAP(tmpmask, NR_CPUS); +static cpumask_var_t tick_broadcast_mask; +static cpumask_var_t tmpmask; static DEFINE_RAW_SPINLOCK(tick_broadcast_lock); static int tick_broadcast_force; @@ -50,7 +49,7 @@ struct tick_device *tick_get_broadcast_device(void) struct cpumask *tick_get_broadcast_mask(void) { - return to_cpumask(tick_broadcast_mask); + return tick_broadcast_mask; } /* @@ -74,7 +73,7 @@ int tick_check_broadcast_device(struct clock_event_device *dev) clockevents_exchange_device(tick_broadcast_device.evtdev, dev); tick_broadcast_device.evtdev = dev; - if (!cpumask_empty(tick_get_broadcast_mask())) + if (!cpumask_empty(tick_broadcast_mask)) tick_broadcast_start_periodic(dev); return 1; } @@ -123,7 +122,7 @@ int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu) if (!tick_device_is_functional(dev)) { dev->event_handler = tick_handle_periodic; tick_device_setup_broadcast_func(dev); - cpumask_set_cpu(cpu, tick_get_broadcast_mask()); + cpumask_set_cpu(cpu, tick_broadcast_mask); tick_broadcast_start_periodic(tick_broadcast_device.evtdev); ret = 1; } else { @@ -134,7 +133,7 @@ int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu) */ if (!(dev->features & CLOCK_EVT_FEAT_C3STOP)) { int cpu = smp_processor_id(); - cpumask_clear_cpu(cpu, tick_get_broadcast_mask()); + cpumask_clear_cpu(cpu, tick_broadcast_mask); tick_broadcast_clear_oneshot(cpu); } else { tick_device_setup_broadcast_func(dev); @@ -198,9 +197,8 @@ static void tick_do_periodic_broadcast(void) { raw_spin_lock(&tick_broadcast_lock); - cpumask_and(to_cpumask(tmpmask), - cpu_online_mask, tick_get_broadcast_mask()); - tick_do_broadcast(to_cpumask(tmpmask)); + cpumask_and(tmpmask, cpu_online_mask, tick_broadcast_mask); + tick_do_broadcast(tmpmask); raw_spin_unlock(&tick_broadcast_lock); } @@ -263,13 +261,12 @@ static void tick_do_broadcast_on_off(unsigned long *reason) if (!tick_device_is_functional(dev)) goto out; - bc_stopped = cpumask_empty(tick_get_broadcast_mask()); + bc_stopped = cpumask_empty(tick_broadcast_mask); switch (*reason) { case CLOCK_EVT_NOTIFY_BROADCAST_ON: case CLOCK_EVT_NOTIFY_BROADCAST_FORCE: - if (!cpumask_test_cpu(cpu, tick_get_broadcast_mask())) { - cpumask_set_cpu(cpu, tick_get_broadcast_mask()); + if (!cpumask_test_and_set_cpu(cpu, tick_broadcast_mask)) { if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) clockevents_shutdown(dev); @@ -279,8 +276,7 @@ static void tick_do_broadcast_on_off(unsigned long *reason) break; case CLOCK_EVT_NOTIFY_BROADCAST_OFF: if (!tick_broadcast_force && - cpumask_test_cpu(cpu, tick_get_broadcast_mask())) { - cpumask_clear_cpu(cpu, tick_get_broadcast_mask()); + cpumask_test_and_clear_cpu(cpu, tick_broadcast_mask)) { if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) tick_setup_periodic(dev, 0); @@ -288,7 +284,7 @@ static void tick_do_broadcast_on_off(unsigned long *reason) break; } - if (cpumask_empty(tick_get_broadcast_mask())) { + if (cpumask_empty(tick_broadcast_mask)) { if (!bc_stopped) clockevents_shutdown(bc); } else if (bc_stopped) { @@ -337,10 +333,10 @@ void tick_shutdown_broadcast(unsigned int *cpup) raw_spin_lock_irqsave(&tick_broadcast_lock, flags); bc = tick_broadcast_device.evtdev; - cpumask_clear_cpu(cpu, tick_get_broadcast_mask()); + cpumask_clear_cpu(cpu, tick_broadcast_mask); if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) { - if (bc && cpumask_empty(tick_get_broadcast_mask())) + if (bc && cpumask_empty(tick_broadcast_mask)) clockevents_shutdown(bc); } @@ -376,13 +372,13 @@ int tick_resume_broadcast(void) switch (tick_broadcast_device.mode) { case TICKDEV_MODE_PERIODIC: - if (!cpumask_empty(tick_get_broadcast_mask())) + if (!cpumask_empty(tick_broadcast_mask)) tick_broadcast_start_periodic(bc); broadcast = cpumask_test_cpu(smp_processor_id(), - tick_get_broadcast_mask()); + tick_broadcast_mask); break; case TICKDEV_MODE_ONESHOT: - if (!cpumask_empty(tick_get_broadcast_mask())) + if (!cpumask_empty(tick_broadcast_mask)) broadcast = tick_resume_broadcast_oneshot(bc); break; } @@ -395,15 +391,14 @@ int tick_resume_broadcast(void) #ifdef CONFIG_TICK_ONESHOT -/* FIXME: use cpumask_var_t. */ -static DECLARE_BITMAP(tick_broadcast_oneshot_mask, NR_CPUS); +static cpumask_var_t tick_broadcast_oneshot_mask; /* * Exposed for debugging: see timer_list.c */ struct cpumask *tick_get_broadcast_oneshot_mask(void) { - return to_cpumask(tick_broadcast_oneshot_mask); + return tick_broadcast_oneshot_mask; } static int tick_broadcast_set_event(ktime_t expires, int force) @@ -428,7 +423,7 @@ int tick_resume_broadcast_oneshot(struct clock_event_device *bc) */ void tick_check_oneshot_broadcast(int cpu) { - if (cpumask_test_cpu(cpu, to_cpumask(tick_broadcast_oneshot_mask))) { + if (cpumask_test_cpu(cpu, tick_broadcast_oneshot_mask)) { struct tick_device *td = &per_cpu(tick_cpu_device, cpu); clockevents_set_mode(td->evtdev, CLOCK_EVT_MODE_ONESHOT); @@ -448,13 +443,13 @@ static void tick_handle_oneshot_broadcast(struct clock_event_device *dev) again: dev->next_event.tv64 = KTIME_MAX; next_event.tv64 = KTIME_MAX; - cpumask_clear(to_cpumask(tmpmask)); + cpumask_clear(tmpmask); now = ktime_get(); /* Find all expired events */ - for_each_cpu(cpu, tick_get_broadcast_oneshot_mask()) { + for_each_cpu(cpu, tick_broadcast_oneshot_mask) { td = &per_cpu(tick_cpu_device, cpu); if (td->evtdev->next_event.tv64 <= now.tv64) - cpumask_set_cpu(cpu, to_cpumask(tmpmask)); + cpumask_set_cpu(cpu, tmpmask); else if (td->evtdev->next_event.tv64 < next_event.tv64) next_event.tv64 = td->evtdev->next_event.tv64; } @@ -462,7 +457,7 @@ again: /* * Wakeup the cpus which have an expired event. */ - tick_do_broadcast(to_cpumask(tmpmask)); + tick_do_broadcast(tmpmask); /* * Two reasons for reprogram: @@ -518,16 +513,13 @@ void tick_broadcast_oneshot_control(unsigned long reason) raw_spin_lock_irqsave(&tick_broadcast_lock, flags); if (reason == CLOCK_EVT_NOTIFY_BROADCAST_ENTER) { - if (!cpumask_test_cpu(cpu, tick_get_broadcast_oneshot_mask())) { - cpumask_set_cpu(cpu, tick_get_broadcast_oneshot_mask()); + if (!cpumask_test_and_set_cpu(cpu, tick_broadcast_oneshot_mask)) { clockevents_set_mode(dev, CLOCK_EVT_MODE_SHUTDOWN); if (dev->next_event.tv64 < bc->next_event.tv64) tick_broadcast_set_event(dev->next_event, 1); } } else { - if (cpumask_test_cpu(cpu, tick_get_broadcast_oneshot_mask())) { - cpumask_clear_cpu(cpu, - tick_get_broadcast_oneshot_mask()); + if (cpumask_test_and_clear_cpu(cpu, tick_broadcast_oneshot_mask)) { clockevents_set_mode(dev, CLOCK_EVT_MODE_ONESHOT); if (dev->next_event.tv64 != KTIME_MAX) tick_program_event(dev->next_event, 1); @@ -543,7 +535,7 @@ void tick_broadcast_oneshot_control(unsigned long reason) */ static void tick_broadcast_clear_oneshot(int cpu) { - cpumask_clear_cpu(cpu, tick_get_broadcast_oneshot_mask()); + cpumask_clear_cpu(cpu, tick_broadcast_oneshot_mask); } static void tick_broadcast_init_next_event(struct cpumask *mask, @@ -581,15 +573,14 @@ void tick_broadcast_setup_oneshot(struct clock_event_device *bc) * oneshot_mask bits for those and program the * broadcast device to fire. */ - cpumask_copy(to_cpumask(tmpmask), tick_get_broadcast_mask()); - cpumask_clear_cpu(cpu, to_cpumask(tmpmask)); - cpumask_or(tick_get_broadcast_oneshot_mask(), - tick_get_broadcast_oneshot_mask(), - to_cpumask(tmpmask)); + cpumask_copy(tmpmask, tick_broadcast_mask); + cpumask_clear_cpu(cpu, tmpmask); + cpumask_or(tick_broadcast_oneshot_mask, + tick_broadcast_oneshot_mask, tmpmask); - if (was_periodic && !cpumask_empty(to_cpumask(tmpmask))) { + if (was_periodic && !cpumask_empty(tmpmask)) { clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT); - tick_broadcast_init_next_event(to_cpumask(tmpmask), + tick_broadcast_init_next_event(tmpmask, tick_next_period); tick_broadcast_set_event(tick_next_period, 1); } else @@ -639,7 +630,7 @@ void tick_shutdown_broadcast_oneshot(unsigned int *cpup) * Clear the broadcast mask flag for the dead cpu, but do not * stop the broadcast device! */ - cpumask_clear_cpu(cpu, tick_get_broadcast_oneshot_mask()); + cpumask_clear_cpu(cpu, tick_broadcast_oneshot_mask); raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags); } @@ -663,3 +654,12 @@ bool tick_broadcast_oneshot_available(void) } #endif + +void __init tick_broadcast_init(void) +{ + alloc_cpumask_var(&tick_broadcast_mask, GFP_NOWAIT); + alloc_cpumask_var(&tmpmask, GFP_NOWAIT); +#ifdef CONFIG_TICK_ONESHOT + alloc_cpumask_var(&tick_broadcast_oneshot_mask, GFP_NOWAIT); +#endif +} diff --git a/kernel/time/tick-common.c b/kernel/time/tick-common.c index b1600a6973f4..74413e396acc 100644 --- a/kernel/time/tick-common.c +++ b/kernel/time/tick-common.c @@ -416,4 +416,5 @@ static struct notifier_block tick_notifier = { void __init tick_init(void) { clockevents_register_notifier(&tick_notifier); + tick_broadcast_init(); } diff --git a/kernel/time/tick-internal.h b/kernel/time/tick-internal.h index cf3e59ed6dc0..46d9bd02844c 100644 --- a/kernel/time/tick-internal.h +++ b/kernel/time/tick-internal.h @@ -94,7 +94,7 @@ extern void tick_broadcast_on_off(unsigned long reason, int *oncpu); extern void tick_shutdown_broadcast(unsigned int *cpup); extern void tick_suspend_broadcast(void); extern int tick_resume_broadcast(void); - +extern void tick_broadcast_init(void); extern void tick_set_periodic_handler(struct clock_event_device *dev, int broadcast); @@ -119,6 +119,7 @@ static inline void tick_broadcast_on_off(unsigned long reason, int *oncpu) { } static inline void tick_shutdown_broadcast(unsigned int *cpup) { } static inline void tick_suspend_broadcast(void) { } static inline int tick_resume_broadcast(void) { return 0; } +static inline void tick_broadcast_init(void) { } /* * Set the periodic handler in non broadcast mode -- cgit v1.2.3 From f9ae39d04ccdec8d8ecf532191b7056c279a22c0 Mon Sep 17 00:00:00 2001 From: Daniel Lezcano Date: Sat, 2 Mar 2013 11:10:10 +0100 Subject: tick: Pass broadcast device to tick_broadcast_set_event() Pass the broadcast timer to tick_broadcast_set_event() instead of reevaluating tick_broadcast_device.evtdev. [ tglx: Massaged changelog ] Signed-off-by: Daniel Lezcano Cc: viresh.kumar@linaro.org Cc: jacob.jun.pan@linux.intel.com Cc: linux-arm-kernel@lists.infradead.org Cc: santosh.shilimkar@ti.com Cc: linaro-kernel@lists.linaro.org Cc: patches@linaro.org Cc: rickard.andersson@stericsson.com Cc: vincent.guittot@linaro.org Cc: linus.walleij@stericsson.com Cc: john.stultz@linaro.org Link: http://lkml.kernel.org/r/1362219013-18173-2-git-send-email-daniel.lezcano@linaro.org Signed-off-by: Thomas Gleixner --- kernel/time/tick-broadcast.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c index 35b887517766..70dd98ce18d7 100644 --- a/kernel/time/tick-broadcast.c +++ b/kernel/time/tick-broadcast.c @@ -401,10 +401,9 @@ struct cpumask *tick_get_broadcast_oneshot_mask(void) return tick_broadcast_oneshot_mask; } -static int tick_broadcast_set_event(ktime_t expires, int force) +static int tick_broadcast_set_event(struct clock_event_device *bc, + ktime_t expires, int force) { - struct clock_event_device *bc = tick_broadcast_device.evtdev; - if (bc->mode != CLOCK_EVT_MODE_ONESHOT) clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT); @@ -474,7 +473,7 @@ again: * Rearm the broadcast device. If event expired, * repeat the above */ - if (tick_broadcast_set_event(next_event, 0)) + if (tick_broadcast_set_event(dev, next_event, 0)) goto again; } raw_spin_unlock(&tick_broadcast_lock); @@ -516,7 +515,7 @@ void tick_broadcast_oneshot_control(unsigned long reason) if (!cpumask_test_and_set_cpu(cpu, tick_broadcast_oneshot_mask)) { clockevents_set_mode(dev, CLOCK_EVT_MODE_SHUTDOWN); if (dev->next_event.tv64 < bc->next_event.tv64) - tick_broadcast_set_event(dev->next_event, 1); + tick_broadcast_set_event(bc, dev->next_event, 1); } } else { if (cpumask_test_and_clear_cpu(cpu, tick_broadcast_oneshot_mask)) { @@ -582,7 +581,7 @@ void tick_broadcast_setup_oneshot(struct clock_event_device *bc) clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT); tick_broadcast_init_next_event(tmpmask, tick_next_period); - tick_broadcast_set_event(tick_next_period, 1); + tick_broadcast_set_event(bc, tick_next_period, 1); } else bc->next_event.tv64 = KTIME_MAX; } else { -- cgit v1.2.3 From d2348fb6fdc6d671ad45b62db237f76c8c115603 Mon Sep 17 00:00:00 2001 From: Daniel Lezcano Date: Sat, 2 Mar 2013 11:10:11 +0100 Subject: tick: Dynamically set broadcast irq affinity When a cpu goes to a deep idle state where its local timer is shutdown, it notifies the time frame work to use the broadcast timer instead. Unfortunately, the broadcast device could wake up any CPU, including an idle one which is not concerned by the wake up at all. So in the worst case an idle CPU will wake up to send an IPI to the CPU whose timer expired. Provide an opt-in feature CLOCK_EVT_FEAT_DYNIRQ which tells the core that is should set the interrupt affinity of the broadcast interrupt to the cpu which has the earliest expiry time. This avoids unnecessary spurious wakeups and IPIs. [ tglx: Adopted to cpumask rework, silenced an uninitialized warning, massaged changelog ] Signed-off-by: Daniel Lezcano Cc: viresh.kumar@linaro.org Cc: jacob.jun.pan@linux.intel.com Cc: linux-arm-kernel@lists.infradead.org Cc: santosh.shilimkar@ti.com Cc: linaro-kernel@lists.linaro.org Cc: patches@linaro.org Cc: rickard.andersson@stericsson.com Cc: vincent.guittot@linaro.org Cc: linus.walleij@stericsson.com Cc: john.stultz@linaro.org Link: http://lkml.kernel.org/r/1362219013-18173-3-git-send-email-daniel.lezcano@linaro.org Signed-off-by: Thomas Gleixner --- include/linux/clockchips.h | 5 +++++ kernel/time/tick-broadcast.c | 39 +++++++++++++++++++++++++++++++-------- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/include/linux/clockchips.h b/include/linux/clockchips.h index 66346521cb65..494d33ea78f8 100644 --- a/include/linux/clockchips.h +++ b/include/linux/clockchips.h @@ -55,6 +55,11 @@ enum clock_event_nofitiers { #define CLOCK_EVT_FEAT_C3STOP 0x000008 #define CLOCK_EVT_FEAT_DUMMY 0x000010 +/* + * Core shall set the interrupt affinity dynamically in broadcast mode + */ +#define CLOCK_EVT_FEAT_DYNIRQ 0x000020 + /** * struct clock_event_device - clock event device descriptor * @event_handler: Assigned by the framework to be called by the low diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c index 70dd98ce18d7..380910db7157 100644 --- a/kernel/time/tick-broadcast.c +++ b/kernel/time/tick-broadcast.c @@ -401,13 +401,34 @@ struct cpumask *tick_get_broadcast_oneshot_mask(void) return tick_broadcast_oneshot_mask; } -static int tick_broadcast_set_event(struct clock_event_device *bc, +/* + * Set broadcast interrupt affinity + */ +static void tick_broadcast_set_affinity(struct clock_event_device *bc, + const struct cpumask *cpumask) +{ + if (!(bc->features & CLOCK_EVT_FEAT_DYNIRQ)) + return; + + if (cpumask_equal(bc->cpumask, cpumask)) + return; + + bc->cpumask = cpumask; + irq_set_affinity(bc->irq, bc->cpumask); +} + +static int tick_broadcast_set_event(struct clock_event_device *bc, int cpu, ktime_t expires, int force) { + int ret; + if (bc->mode != CLOCK_EVT_MODE_ONESHOT) clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT); - return clockevents_program_event(bc, expires, force); + ret = clockevents_program_event(bc, expires, force); + if (!ret) + tick_broadcast_set_affinity(bc, cpumask_of(cpu)); + return ret; } int tick_resume_broadcast_oneshot(struct clock_event_device *bc) @@ -436,7 +457,7 @@ static void tick_handle_oneshot_broadcast(struct clock_event_device *dev) { struct tick_device *td; ktime_t now, next_event; - int cpu; + int cpu, next_cpu = 0; raw_spin_lock(&tick_broadcast_lock); again: @@ -447,10 +468,12 @@ again: /* Find all expired events */ for_each_cpu(cpu, tick_broadcast_oneshot_mask) { td = &per_cpu(tick_cpu_device, cpu); - if (td->evtdev->next_event.tv64 <= now.tv64) + if (td->evtdev->next_event.tv64 <= now.tv64) { cpumask_set_cpu(cpu, tmpmask); - else if (td->evtdev->next_event.tv64 < next_event.tv64) + } else if (td->evtdev->next_event.tv64 < next_event.tv64) { next_event.tv64 = td->evtdev->next_event.tv64; + next_cpu = cpu; + } } /* @@ -473,7 +496,7 @@ again: * Rearm the broadcast device. If event expired, * repeat the above */ - if (tick_broadcast_set_event(dev, next_event, 0)) + if (tick_broadcast_set_event(dev, next_cpu, next_event, 0)) goto again; } raw_spin_unlock(&tick_broadcast_lock); @@ -515,7 +538,7 @@ void tick_broadcast_oneshot_control(unsigned long reason) if (!cpumask_test_and_set_cpu(cpu, tick_broadcast_oneshot_mask)) { clockevents_set_mode(dev, CLOCK_EVT_MODE_SHUTDOWN); if (dev->next_event.tv64 < bc->next_event.tv64) - tick_broadcast_set_event(bc, dev->next_event, 1); + tick_broadcast_set_event(bc, cpu, dev->next_event, 1); } } else { if (cpumask_test_and_clear_cpu(cpu, tick_broadcast_oneshot_mask)) { @@ -581,7 +604,7 @@ void tick_broadcast_setup_oneshot(struct clock_event_device *bc) clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT); tick_broadcast_init_next_event(tmpmask, tick_next_period); - tick_broadcast_set_event(bc, tick_next_period, 1); + tick_broadcast_set_event(bc, cpu, tick_next_period, 1); } else bc->next_event.tv64 = KTIME_MAX; } else { -- cgit v1.2.3 From 26517f3e99248668315aee9460dcea21628cdd7f Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 6 Mar 2013 11:18:35 +0000 Subject: tick: Avoid programming the local cpu timer if broadcast pending If the local cpu timer stops in deep idle, we arm the broadcast device and get woken by an IPI. Now when we return from deep idle we reenable the local cpu timer unconditionally before handling the IPI. But that's a pointless exercise: the timer is already expired and the IPI is on the way. And it's an expensive exercise as we use the forced reprogramming mode so that we do not lose a timer event. This forced reprogramming will loop at least once in the retry. To avoid this reprogramming, we mark the cpu in a pending bit mask before we send the IPI. Now when the IPI target cpu wakes up, it will see the pending bit set and skip the reprogramming. The reprogramming of the cpu local timer will happen in the IPI handler which runs the cpu local timer interrupt function. Reported-by: Jason Liu Signed-off-by: Thomas Gleixner Cc: LAK Cc: John Stultz Cc: Arjan van de Veen Cc: Lorenzo Pieralisi Tested-by: Santosh Shilimkar Link: http://lkml.kernel.org/r/20130306111537.431082074@linutronix.de Signed-off-by: Thomas Gleixner --- kernel/time/tick-broadcast.c | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c index 380910db7157..005c0ca81a32 100644 --- a/kernel/time/tick-broadcast.c +++ b/kernel/time/tick-broadcast.c @@ -392,6 +392,7 @@ int tick_resume_broadcast(void) #ifdef CONFIG_TICK_ONESHOT static cpumask_var_t tick_broadcast_oneshot_mask; +static cpumask_var_t tick_broadcast_pending_mask; /* * Exposed for debugging: see timer_list.c @@ -470,6 +471,12 @@ again: td = &per_cpu(tick_cpu_device, cpu); if (td->evtdev->next_event.tv64 <= now.tv64) { cpumask_set_cpu(cpu, tmpmask); + /* + * Mark the remote cpu in the pending mask, so + * it can avoid reprogramming the cpu local + * timer in tick_broadcast_oneshot_control(). + */ + cpumask_set_cpu(cpu, tick_broadcast_pending_mask); } else if (td->evtdev->next_event.tv64 < next_event.tv64) { next_event.tv64 = td->evtdev->next_event.tv64; next_cpu = cpu; @@ -535,6 +542,7 @@ void tick_broadcast_oneshot_control(unsigned long reason) raw_spin_lock_irqsave(&tick_broadcast_lock, flags); if (reason == CLOCK_EVT_NOTIFY_BROADCAST_ENTER) { + WARN_ON_ONCE(cpumask_test_cpu(cpu, tick_broadcast_pending_mask)); if (!cpumask_test_and_set_cpu(cpu, tick_broadcast_oneshot_mask)) { clockevents_set_mode(dev, CLOCK_EVT_MODE_SHUTDOWN); if (dev->next_event.tv64 < bc->next_event.tv64) @@ -543,10 +551,25 @@ void tick_broadcast_oneshot_control(unsigned long reason) } else { if (cpumask_test_and_clear_cpu(cpu, tick_broadcast_oneshot_mask)) { clockevents_set_mode(dev, CLOCK_EVT_MODE_ONESHOT); - if (dev->next_event.tv64 != KTIME_MAX) - tick_program_event(dev->next_event, 1); + if (dev->next_event.tv64 == KTIME_MAX) + goto out; + /* + * The cpu which was handling the broadcast + * timer marked this cpu in the broadcast + * pending mask and fired the broadcast + * IPI. So we are going to handle the expired + * event anyway via the broadcast IPI + * handler. No need to reprogram the timer + * with an already expired event. + */ + if (cpumask_test_and_clear_cpu(cpu, + tick_broadcast_pending_mask)) + goto out; + + tick_program_event(dev->next_event, 1); } } +out: raw_spin_unlock_irqrestore(&tick_broadcast_lock, flags); } @@ -683,5 +706,6 @@ void __init tick_broadcast_init(void) alloc_cpumask_var(&tmpmask, GFP_NOWAIT); #ifdef CONFIG_TICK_ONESHOT alloc_cpumask_var(&tick_broadcast_oneshot_mask, GFP_NOWAIT); + alloc_cpumask_var(&tick_broadcast_pending_mask, GFP_NOWAIT); #endif } -- cgit v1.2.3 From 989dcb645ca715129c5a2b39102c8334a20d9615 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 6 Mar 2013 11:18:35 +0000 Subject: tick: Handle broadcast wakeup of multiple cpus Some brilliant hardware implementations wake multiple cores when the broadcast timer fires. This leads to the following interesting problem: CPU0 CPU1 wakeup from idle wakeup from idle leave broadcast mode leave broadcast mode restart per cpu timer restart per cpu timer go back to idle handle broadcast (empty mask) enter broadcast mode programm broadcast device enter broadcast mode programm broadcast device So what happens is that due to the forced reprogramming of the cpu local timer, we need to set a event in the future. Now if we manage to go back to idle before the timer fires, we switch off the timer and arm the broadcast device with an already expired time (covered by forced mode). So in the worst case we repeat the above ping pong forever. Unfortunately we have no information about what caused the wakeup, but we can check current time against the expiry time of the local cpu. If the local event is already in the past, we know that the broadcast timer is about to fire and send an IPI. So we mark ourself as an IPI target even if we left broadcast mode and avoid the reprogramming of the local cpu timer. This still leaves the possibility that a CPU which is not handling the broadcast interrupt is going to reach idle again before the IPI arrives. This can't be solved in the core code and will be handled in follow up patches. Reported-by: Jason Liu Signed-off-by: Thomas Gleixner Cc: LAK Cc: John Stultz Cc: Arjan van de Veen Cc: Lorenzo Pieralisi Tested-by: Santosh Shilimkar Link: http://lkml.kernel.org/r/20130306111537.492045206@linutronix.de Signed-off-by: Thomas Gleixner --- kernel/time/tick-broadcast.c | 59 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c index 005c0ca81a32..2100aad6b5f2 100644 --- a/kernel/time/tick-broadcast.c +++ b/kernel/time/tick-broadcast.c @@ -393,6 +393,7 @@ int tick_resume_broadcast(void) static cpumask_var_t tick_broadcast_oneshot_mask; static cpumask_var_t tick_broadcast_pending_mask; +static cpumask_var_t tick_broadcast_force_mask; /* * Exposed for debugging: see timer_list.c @@ -483,6 +484,10 @@ again: } } + /* Take care of enforced broadcast requests */ + cpumask_or(tmpmask, tmpmask, tick_broadcast_force_mask); + cpumask_clear(tick_broadcast_force_mask); + /* * Wakeup the cpus which have an expired event. */ @@ -518,6 +523,7 @@ void tick_broadcast_oneshot_control(unsigned long reason) struct clock_event_device *bc, *dev; struct tick_device *td; unsigned long flags; + ktime_t now; int cpu; /* @@ -545,7 +551,16 @@ void tick_broadcast_oneshot_control(unsigned long reason) WARN_ON_ONCE(cpumask_test_cpu(cpu, tick_broadcast_pending_mask)); if (!cpumask_test_and_set_cpu(cpu, tick_broadcast_oneshot_mask)) { clockevents_set_mode(dev, CLOCK_EVT_MODE_SHUTDOWN); - if (dev->next_event.tv64 < bc->next_event.tv64) + /* + * We only reprogram the broadcast timer if we + * did not mark ourself in the force mask and + * if the cpu local event is earlier than the + * broadcast event. If the current CPU is in + * the force mask, then we are going to be + * woken by the IPI right away. + */ + if (!cpumask_test_cpu(cpu, tick_broadcast_force_mask) && + dev->next_event.tv64 < bc->next_event.tv64) tick_broadcast_set_event(bc, cpu, dev->next_event, 1); } } else { @@ -566,6 +581,47 @@ void tick_broadcast_oneshot_control(unsigned long reason) tick_broadcast_pending_mask)) goto out; + /* + * If the pending bit is not set, then we are + * either the CPU handling the broadcast + * interrupt or we got woken by something else. + * + * We are not longer in the broadcast mask, so + * if the cpu local expiry time is already + * reached, we would reprogram the cpu local + * timer with an already expired event. + * + * This can lead to a ping-pong when we return + * to idle and therefor rearm the broadcast + * timer before the cpu local timer was able + * to fire. This happens because the forced + * reprogramming makes sure that the event + * will happen in the future and depending on + * the min_delta setting this might be far + * enough out that the ping-pong starts. + * + * If the cpu local next_event has expired + * then we know that the broadcast timer + * next_event has expired as well and + * broadcast is about to be handled. So we + * avoid reprogramming and enforce that the + * broadcast handler, which did not run yet, + * will invoke the cpu local handler. + * + * We cannot call the handler directly from + * here, because we might be in a NOHZ phase + * and we did not go through the irq_enter() + * nohz fixups. + */ + now = ktime_get(); + if (dev->next_event.tv64 <= now.tv64) { + cpumask_set_cpu(cpu, tick_broadcast_force_mask); + goto out; + } + /* + * We got woken by something else. Reprogram + * the cpu local timer device. + */ tick_program_event(dev->next_event, 1); } } @@ -707,5 +763,6 @@ void __init tick_broadcast_init(void) #ifdef CONFIG_TICK_ONESHOT alloc_cpumask_var(&tick_broadcast_oneshot_mask, GFP_NOWAIT); alloc_cpumask_var(&tick_broadcast_pending_mask, GFP_NOWAIT); + alloc_cpumask_var(&tick_broadcast_force_mask, GFP_NOWAIT); #endif } -- cgit v1.2.3 From eaa907c546f76222227dfc41784b22588af1e3d7 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 6 Mar 2013 11:18:36 +0000 Subject: tick: Provide a check for a forced broadcast pending On the CPU which gets woken along with the target CPU of the broadcast the following happens: deep_idle() <-- spurious wakeup broadcast_exit() set forced bit enable interrupts <-- Nothing happens disable interrupts broadcast_enter() <-- Here we observe the forced bit is set deep_idle() Now after that the target CPU of the broadcast runs the broadcast handler and finds the other CPU in both the broadcast and the forced mask, sends the IPI and stuff gets back to normal. So it's not actually harmful, just more evidence for the theory, that hardware designers have access to very special drug supplies. Now there is no point in going back to deep idle just to wake up again right away via an IPI. Provide a check which allows the idle code to avoid the deep idle transition. Signed-off-by: Thomas Gleixner Cc: LAK Cc: John Stultz Cc: Arjan van de Veen Cc: Lorenzo Pieralisi Tested-by: Santosh Shilimkar Cc: Jason Liu Link: http://lkml.kernel.org/r/20130306111537.565418308@linutronix.de Signed-off-by: Thomas Gleixner --- include/linux/clockchips.h | 6 ++++++ kernel/time/tick-broadcast.c | 12 ++++++++++++ 2 files changed, 18 insertions(+) diff --git a/include/linux/clockchips.h b/include/linux/clockchips.h index 494d33ea78f8..646aac136eed 100644 --- a/include/linux/clockchips.h +++ b/include/linux/clockchips.h @@ -175,6 +175,12 @@ extern void tick_broadcast(const struct cpumask *mask); extern int tick_receive_broadcast(void); #endif +#if defined(CONFIG_GENERIC_CLOCKEVENTS_BROADCAST) && defined(CONFIG_TICK_ONESHOT) +extern int tick_check_broadcast_expired(void); +#else +static inline int tick_check_broadcast_expired(void) { return 0; } +#endif + #ifdef CONFIG_GENERIC_CLOCKEVENTS extern void clockevents_notify(unsigned long reason, void *arg); #else diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c index 2100aad6b5f2..d76d816afc5d 100644 --- a/kernel/time/tick-broadcast.c +++ b/kernel/time/tick-broadcast.c @@ -403,6 +403,18 @@ struct cpumask *tick_get_broadcast_oneshot_mask(void) return tick_broadcast_oneshot_mask; } +/* + * Called before going idle with interrupts disabled. Checks whether a + * broadcast event from the other core is about to happen. We detected + * that in tick_broadcast_oneshot_control(). The callsite can use this + * to avoid a deep idle transition as we are about to get the + * broadcast IPI right away. + */ +int tick_check_broadcast_expired(void) +{ + return cpumask_test_cpu(smp_processor_id(), tick_broadcast_force_mask); +} + /* * Set broadcast interrupt affinity */ -- cgit v1.2.3 From 80bbe9f273a38f83ecfc50fe384a57f8428887bd Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 6 Mar 2013 11:18:37 +0000 Subject: arm: Use tick broadcast expired check Avoid going back into deep idle if the tick broadcast IPI is about to fire. Signed-off-by: Thomas Gleixner Cc: LAK Cc: John Stultz Cc: Arjan van de Veen Cc: Lorenzo Pieralisi Tested-by: Santosh Shilimkar Cc: Jason Liu Link: http://lkml.kernel.org/r/20130306111537.640722922@linutronix.de --- arch/arm/kernel/process.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c index 047d3e40e470..db4ffd09ee23 100644 --- a/arch/arm/kernel/process.c +++ b/arch/arm/kernel/process.c @@ -199,7 +199,16 @@ void cpu_idle(void) #ifdef CONFIG_PL310_ERRATA_769419 wmb(); #endif - if (hlt_counter) { + /* + * In poll mode we reenable interrupts and spin. + * + * Also if we detected in the wakeup from idle + * path that the tick broadcast device expired + * for us, we don't want to go deep idle as we + * know that the IPI is going to arrive right + * away + */ + if (hlt_counter || tick_check_broadcast_expired()) { local_irq_enable(); cpu_relax(); } else if (!need_resched()) { -- cgit v1.2.3 From 35b61edb41ffee58711850e76215b852386ddb10 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Wed, 6 Mar 2013 11:18:37 +0000 Subject: x86: Use tick broadcast expired check Avoid going back into deep idle if the tick broadcast IPI is about to fire. Signed-off-by: Thomas Gleixner Cc: John Stultz Cc: Arjan van de Veen Cc: x86@kernel.org Link: http://lkml.kernel.org/r/20130306111537.702278273@linutronix.de --- arch/x86/kernel/process.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index 14ae10031ff0..aa524da03bba 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -336,6 +336,18 @@ void cpu_idle(void) local_touch_nmi(); local_irq_disable(); + /* + * We detected in the wakeup path that the + * tick broadcast device expired for us, but + * we raced with the other CPU and came back + * here before it was able to fire the IPI. + * No point in going idle. + */ + if (tick_check_broadcast_expired()) { + local_irq_enable(); + continue; + } + enter_idle(); /* Don't trace irqs off for idle */ -- cgit v1.2.3 From c30bd09915ea243603d7803d53de890c4a6f1474 Mon Sep 17 00:00:00 2001 From: Dong Zhu Date: Thu, 6 Dec 2012 22:03:34 +0800 Subject: timekeeping: Avoid adjust kernel time once hwclock kept in UTC time If the Hardware Clock kept in local time,kernel will adjust the time to be UTC time.But if Hardware Clock kept in UTC time,system will make a dummy settimeofday call first (sys_tz.tz_minuteswest = 0) to make sure the time is not shifted,so at this point I think maybe it is not necessary to set the kernel time once the sys_tz.tz_minuteswest is zero. Signed-off-by: Dong Zhu [jstultz: Updated to merge with conflicting changes ] Signed-off-by: John Stultz --- kernel/time.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/kernel/time.c b/kernel/time.c index f8342a41efa6..effac571589f 100644 --- a/kernel/time.c +++ b/kernel/time.c @@ -138,13 +138,14 @@ int persistent_clock_is_local; */ static inline void warp_clock(void) { - struct timespec adjust; + if (sys_tz.tz_minuteswest != 0) { + struct timespec adjust; - adjust = current_kernel_time(); - if (sys_tz.tz_minuteswest != 0) persistent_clock_is_local = 1; - adjust.tv_sec += sys_tz.tz_minuteswest * 60; - do_settimeofday(&adjust); + adjust = current_kernel_time(); + adjust.tv_sec += sys_tz.tz_minuteswest * 60; + do_settimeofday(&adjust); + } } /* -- cgit v1.2.3 From 7859e404ae73fe4f38b8cfc1af19ea82f153084e Mon Sep 17 00:00:00 2001 From: John Stultz Date: Fri, 22 Feb 2013 12:33:29 -0800 Subject: timekeeping: Use inject_offset in warp_clock When warping the clock (from a local time RTC), use timekeeping_inject_offset() to atomically add the offset. This avoids any minor time error caused by the delay between reading the time, and then setting the adjusted time. Signed-off-by: John Stultz --- kernel/time.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kernel/time.c b/kernel/time.c index effac571589f..d3617dbd3dca 100644 --- a/kernel/time.c +++ b/kernel/time.c @@ -142,9 +142,9 @@ static inline void warp_clock(void) struct timespec adjust; persistent_clock_is_local = 1; - adjust = current_kernel_time(); - adjust.tv_sec += sys_tz.tz_minuteswest * 60; - do_settimeofday(&adjust); + adjust.tv_sec = sys_tz.tz_minuteswest * 60; + adjust.tv_nsec = 0; + timekeeping_inject_offset(&adjust); } } -- cgit v1.2.3 From 3195ef59cb42cda3aeeb24a7fd2ba1b900c4a3cc Mon Sep 17 00:00:00 2001 From: Prarit Bhargava Date: Thu, 14 Feb 2013 12:02:54 -0500 Subject: x86: Do full rtc synchronization with ntp Every 11 minutes ntp attempts to update the x86 rtc with the current system time. Currently, the x86 code only updates the rtc if the system time is within +/-15 minutes of the current value of the rtc. This was done originally to avoid setting the RTC if the RTC was in localtime mode (common with Windows dualbooting). Other architectures do a full synchronization and now that we have better infrastructure to detect when the RTC is in localtime, there is no reason that x86 should be software limited to a 30 minute window. This patch changes the behavior of the kernel to do a full synchronization (year, month, day, hour, minute, and second) of the rtc when ntp requests a synchronization between the system time and the rtc. I've used the RTC library functions in this patchset as they do all the required bounds checking. Cc: Thomas Gleixner Cc: John Stultz Cc: x86@kernel.org Cc: Matt Fleming Cc: David Vrabel Cc: Andrew Morton Cc: Andi Kleen Cc: linux-efi@vger.kernel.org Signed-off-by: Prarit Bhargava [jstultz: Tweak commit message, fold in build fix found by fengguang Also add select RTC_LIB to X86, per new dependency, as found by prarit] Signed-off-by: John Stultz --- arch/x86/Kconfig | 1 + arch/x86/kernel/rtc.c | 69 ++++++++----------------------------------- arch/x86/platform/efi/efi.c | 24 ++++++++++----- arch/x86/platform/mrst/vrtc.c | 44 ++++++++++++++++----------- 4 files changed, 55 insertions(+), 83 deletions(-) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index a4f24f5b1218..26bd79261532 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -120,6 +120,7 @@ config X86 select OLD_SIGSUSPEND3 if X86_32 || IA32_EMULATION select OLD_SIGACTION if X86_32 select COMPAT_OLD_SIGACTION if IA32_EMULATION + select RTC_LIB config INSTRUCTION_DECODER def_bool y diff --git a/arch/x86/kernel/rtc.c b/arch/x86/kernel/rtc.c index 2e8f3d3b5641..198eb201ed3b 100644 --- a/arch/x86/kernel/rtc.c +++ b/arch/x86/kernel/rtc.c @@ -13,6 +13,7 @@ #include #include #include +#include #ifdef CONFIG_X86_32 /* @@ -36,70 +37,24 @@ EXPORT_SYMBOL(rtc_lock); * nowtime is written into the registers of the CMOS clock, it will * jump to the next second precisely 500 ms later. Check the Motorola * MC146818A or Dallas DS12887 data sheet for details. - * - * BUG: This routine does not handle hour overflow properly; it just - * sets the minutes. Usually you'll only notice that after reboot! */ int mach_set_rtc_mmss(unsigned long nowtime) { - int real_seconds, real_minutes, cmos_minutes; - unsigned char save_control, save_freq_select; - unsigned long flags; + struct rtc_time tm; int retval = 0; - spin_lock_irqsave(&rtc_lock, flags); - - /* tell the clock it's being set */ - save_control = CMOS_READ(RTC_CONTROL); - CMOS_WRITE((save_control|RTC_SET), RTC_CONTROL); - - /* stop and reset prescaler */ - save_freq_select = CMOS_READ(RTC_FREQ_SELECT); - CMOS_WRITE((save_freq_select|RTC_DIV_RESET2), RTC_FREQ_SELECT); - - cmos_minutes = CMOS_READ(RTC_MINUTES); - if (!(save_control & RTC_DM_BINARY) || RTC_ALWAYS_BCD) - cmos_minutes = bcd2bin(cmos_minutes); - - /* - * since we're only adjusting minutes and seconds, - * don't interfere with hour overflow. This avoids - * messing with unknown time zones but requires your - * RTC not to be off by more than 15 minutes - */ - real_seconds = nowtime % 60; - real_minutes = nowtime / 60; - /* correct for half hour time zone */ - if (((abs(real_minutes - cmos_minutes) + 15)/30) & 1) - real_minutes += 30; - real_minutes %= 60; - - if (abs(real_minutes - cmos_minutes) < 30) { - if (!(save_control & RTC_DM_BINARY) || RTC_ALWAYS_BCD) { - real_seconds = bin2bcd(real_seconds); - real_minutes = bin2bcd(real_minutes); - } - CMOS_WRITE(real_seconds, RTC_SECONDS); - CMOS_WRITE(real_minutes, RTC_MINUTES); + rtc_time_to_tm(nowtime, &tm); + if (!rtc_valid_tm(&tm)) { + retval = set_rtc_time(&tm); + if (retval) + printk(KERN_ERR "%s: RTC write failed with error %d\n", + __FUNCTION__, retval); } else { - printk_once(KERN_NOTICE - "set_rtc_mmss: can't update from %d to %d\n", - cmos_minutes, real_minutes); - retval = -1; + printk(KERN_ERR + "%s: Invalid RTC value: write of %lx to RTC failed\n", + __FUNCTION__, nowtime); + retval = -EINVAL; } - - /* The following flags have to be released exactly in this order, - * otherwise the DS12887 (popular MC146818A clone with integrated - * battery and quartz) will not reset the oscillator and will not - * update precisely 500 ms later. You won't find this mentioned in - * the Dallas Semiconductor data sheets, but who believes data - * sheets anyway ... -- Markus Kuhn - */ - CMOS_WRITE(save_control, RTC_CONTROL); - CMOS_WRITE(save_freq_select, RTC_FREQ_SELECT); - - spin_unlock_irqrestore(&rtc_lock, flags); - return retval; } diff --git a/arch/x86/platform/efi/efi.c b/arch/x86/platform/efi/efi.c index 5f2ecaf3f9d8..28d9efacc9b6 100644 --- a/arch/x86/platform/efi/efi.c +++ b/arch/x86/platform/efi/efi.c @@ -48,6 +48,7 @@ #include #include #include +#include #define EFI_DEBUG 1 @@ -258,10 +259,10 @@ static efi_status_t __init phys_efi_get_time(efi_time_t *tm, int efi_set_rtc_mmss(unsigned long nowtime) { - int real_seconds, real_minutes; efi_status_t status; efi_time_t eft; efi_time_cap_t cap; + struct rtc_time tm; status = efi.get_time(&eft, &cap); if (status != EFI_SUCCESS) { @@ -269,13 +270,20 @@ int efi_set_rtc_mmss(unsigned long nowtime) return -1; } - real_seconds = nowtime % 60; - real_minutes = nowtime / 60; - if (((abs(real_minutes - eft.minute) + 15)/30) & 1) - real_minutes += 30; - real_minutes %= 60; - eft.minute = real_minutes; - eft.second = real_seconds; + rtc_time_to_tm(nowtime, &tm); + if (!rtc_valid_tm(&tm)) { + eft.year = tm.tm_year + 1900; + eft.month = tm.tm_mon + 1; + eft.day = tm.tm_mday; + eft.minute = tm.tm_min; + eft.second = tm.tm_sec; + eft.nanosecond = 0; + } else { + printk(KERN_ERR + "%s: Invalid EFI RTC value: write of %lx to EFI RTC failed\n", + __FUNCTION__, nowtime); + return -1; + } status = efi.set_time(&eft); if (status != EFI_SUCCESS) { diff --git a/arch/x86/platform/mrst/vrtc.c b/arch/x86/platform/mrst/vrtc.c index 225bd0f0f675..d62b0a3b5c14 100644 --- a/arch/x86/platform/mrst/vrtc.c +++ b/arch/x86/platform/mrst/vrtc.c @@ -85,27 +85,35 @@ unsigned long vrtc_get_time(void) return mktime(year, mon, mday, hour, min, sec); } -/* Only care about the minutes and seconds */ int vrtc_set_mmss(unsigned long nowtime) { - int real_sec, real_min; unsigned long flags; - int vrtc_min; - - spin_lock_irqsave(&rtc_lock, flags); - vrtc_min = vrtc_cmos_read(RTC_MINUTES); - - real_sec = nowtime % 60; - real_min = nowtime / 60; - if (((abs(real_min - vrtc_min) + 15)/30) & 1) - real_min += 30; - real_min %= 60; - - vrtc_cmos_write(real_sec, RTC_SECONDS); - vrtc_cmos_write(real_min, RTC_MINUTES); - spin_unlock_irqrestore(&rtc_lock, flags); - - return 0; + struct rtc_time tm; + int year; + int retval = 0; + + rtc_time_to_tm(nowtime, &tm); + if (!rtc_valid_tm(&tm) && tm.tm_year >= 72) { + /* + * tm.year is the number of years since 1900, and the + * vrtc need the years since 1972. + */ + year = tm.tm_year - 72; + spin_lock_irqsave(&rtc_lock, flags); + vrtc_cmos_write(year, RTC_YEAR); + vrtc_cmos_write(tm.tm_mon, RTC_MONTH); + vrtc_cmos_write(tm.tm_mday, RTC_DAY_OF_MONTH); + vrtc_cmos_write(tm.tm_hour, RTC_HOURS); + vrtc_cmos_write(tm.tm_min, RTC_MINUTES); + vrtc_cmos_write(tm.tm_sec, RTC_SECONDS); + spin_unlock_irqrestore(&rtc_lock, flags); + } else { + printk(KERN_ERR + "%s: Invalid vRTC value: write of %lx to vRTC failed\n", + __FUNCTION__, nowtime); + retval = -EINVAL; + } + return retval; } void __init mrst_rtc_init(void) -- cgit v1.2.3 From c54fdbb2823d96b842d00c548e14dbc0dd37831d Mon Sep 17 00:00:00 2001 From: Feng Tang Date: Tue, 12 Mar 2013 11:56:45 +0800 Subject: x86: Add cpu capability flag X86_FEATURE_NONSTOP_TSC_S3 On some new Intel Atom processors (Penwell and Cloverview), there is a feature that the TSC won't stop in S3 state, say the TSC value won't be reset to 0 after resume. This feature makes TSC a more reliable clocksource and could benefit the timekeeping code during system suspend/resume cycle, so add a flag for it. Signed-off-by: Feng Tang [jstultz: Fix checkpatch warning] Signed-off-by: John Stultz --- arch/x86/include/asm/cpufeature.h | 1 + arch/x86/kernel/cpu/intel.c | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h index 93fe929d1cee..a8466f203e62 100644 --- a/arch/x86/include/asm/cpufeature.h +++ b/arch/x86/include/asm/cpufeature.h @@ -100,6 +100,7 @@ #define X86_FEATURE_AMD_DCM (3*32+27) /* multi-node processor */ #define X86_FEATURE_APERFMPERF (3*32+28) /* APERFMPERF */ #define X86_FEATURE_EAGER_FPU (3*32+29) /* "eagerfpu" Non lazy FPU restore */ +#define X86_FEATURE_NONSTOP_TSC_S3 (3*32+30) /* TSC doesn't stop in S3 state */ /* Intel-defined CPU features, CPUID level 0x00000001 (ecx), word 4 */ #define X86_FEATURE_XMM3 (4*32+ 0) /* "pni" SSE-3 */ diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c index 1905ce98bee0..e7ae0d89e7e0 100644 --- a/arch/x86/kernel/cpu/intel.c +++ b/arch/x86/kernel/cpu/intel.c @@ -96,6 +96,18 @@ static void __cpuinit early_init_intel(struct cpuinfo_x86 *c) sched_clock_stable = 1; } + /* Penwell and Cloverview have the TSC which doesn't sleep on S3 */ + if (c->x86 == 6) { + switch (c->x86_model) { + case 0x27: /* Penwell */ + case 0x35: /* Cloverview */ + set_cpu_cap(c, X86_FEATURE_NONSTOP_TSC_S3); + break; + default: + break; + } + } + /* * There is a known erratum on Pentium III and Core Solo * and Core Duo CPUs. -- cgit v1.2.3 From 5caf4636259ae3af0efbb9bfc4cd97874b547c7d Mon Sep 17 00:00:00 2001 From: Feng Tang Date: Tue, 12 Mar 2013 11:56:46 +0800 Subject: clocksource: Add new feature flag CLOCK_SOURCE_SUSPEND_NONSTOP Some x86 processors have a TSC clocksource, which continues to run even when system is suspended. Also most OMAP platforms have a 32 KHz timer which has similar capability. Add a feature flag so that it could be utilized. Signed-off-by: Feng Tang Signed-off-by: John Stultz --- include/linux/clocksource.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/linux/clocksource.h b/include/linux/clocksource.h index 27cfda427dd9..aa7032c7238f 100644 --- a/include/linux/clocksource.h +++ b/include/linux/clocksource.h @@ -206,6 +206,7 @@ struct clocksource { #define CLOCK_SOURCE_WATCHDOG 0x10 #define CLOCK_SOURCE_VALID_FOR_HRES 0x20 #define CLOCK_SOURCE_UNSTABLE 0x40 +#define CLOCK_SOURCE_SUSPEND_NONSTOP 0x80 /* simplify initialization of mask field */ #define CLOCKSOURCE_MASK(bits) (cycle_t)((bits) < 64 ? ((1ULL<<(bits))-1) : -1) -- cgit v1.2.3 From 82f9c080b22a5b859ae2b50822dfb6b812898fdb Mon Sep 17 00:00:00 2001 From: Feng Tang Date: Tue, 12 Mar 2013 11:56:47 +0800 Subject: x86: tsc: Add support for new S3_NONSTOP feature Add support for new S3_NONSTOP feature Signed-off-by: Feng Tang Signed-off-by: John Stultz --- arch/x86/kernel/tsc.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c index 4b9ea101fe3b..098b3cfda72e 100644 --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c @@ -768,7 +768,8 @@ static cycle_t read_tsc(struct clocksource *cs) static void resume_tsc(struct clocksource *cs) { - clocksource_tsc.cycle_last = 0; + if (!boot_cpu_has(X86_FEATURE_NONSTOP_TSC_S3)) + clocksource_tsc.cycle_last = 0; } static struct clocksource clocksource_tsc = { @@ -939,6 +940,9 @@ static int __init init_tsc_clocksource(void) clocksource_tsc.flags &= ~CLOCK_SOURCE_IS_CONTINUOUS; } + if (boot_cpu_has(X86_FEATURE_NONSTOP_TSC_S3)) + clocksource_tsc.flags |= CLOCK_SOURCE_SUSPEND_NONSTOP; + /* * Trust the results of the earlier calibration on systems * exporting a reliable TSC. -- cgit v1.2.3 From e445cf1c4257cc0238d72e4129eb4739f46fd3de Mon Sep 17 00:00:00 2001 From: Feng Tang Date: Tue, 12 Mar 2013 11:56:48 +0800 Subject: timekeeping: utilize the suspend-nonstop clocksource to count suspended time There are some new processors whose TSC clocksource won't stop during suspend. Currently, after system resumes, kernel will use persistent clock or RTC to compensate the sleep time, but with these nonstop clocksources, we could skip the special compensation from external sources, and just use current clocksource for time recounting. This can solve some time drift bugs caused by some not-so-accurate or error-prone RTC devices. The current way to count suspended time is first try to use the persistent clock, and then try the RTC if persistent clock can't be used. This patch will change the trying order to: suspend-nonstop clocksource -> persistent clock -> RTC When counting the sleep time with nonstop clocksource, use an accurate way suggested by Jason Gunthorpe to cover very large delta cycles. Signed-off-by: Feng Tang [jstultz: Small optimization, avoiding re-reading the clocksource] Signed-off-by: John Stultz --- kernel/time/timekeeping.c | 58 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 51 insertions(+), 7 deletions(-) diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 9a0bc98fbe1d..0355f125d585 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -788,22 +788,66 @@ void timekeeping_inject_sleeptime(struct timespec *delta) static void timekeeping_resume(void) { struct timekeeper *tk = &timekeeper; + struct clocksource *clock = tk->clock; unsigned long flags; - struct timespec ts; + struct timespec ts_new, ts_delta; + cycle_t cycle_now, cycle_delta; + bool suspendtime_found = false; - read_persistent_clock(&ts); + read_persistent_clock(&ts_new); clockevents_resume(); clocksource_resume(); write_seqlock_irqsave(&tk->lock, flags); - if (timespec_compare(&ts, &timekeeping_suspend_time) > 0) { - ts = timespec_sub(ts, timekeeping_suspend_time); - __timekeeping_inject_sleeptime(tk, &ts); + /* + * After system resumes, we need to calculate the suspended time and + * compensate it for the OS time. There are 3 sources that could be + * used: Nonstop clocksource during suspend, persistent clock and rtc + * device. + * + * One specific platform may have 1 or 2 or all of them, and the + * preference will be: + * suspend-nonstop clocksource -> persistent clock -> rtc + * The less preferred source will only be tried if there is no better + * usable source. The rtc part is handled separately in rtc core code. + */ + cycle_now = clock->read(clock); + if ((clock->flags & CLOCK_SOURCE_SUSPEND_NONSTOP) && + cycle_now > clock->cycle_last) { + u64 num, max = ULLONG_MAX; + u32 mult = clock->mult; + u32 shift = clock->shift; + s64 nsec = 0; + + cycle_delta = (cycle_now - clock->cycle_last) & clock->mask; + + /* + * "cycle_delta * mutl" may cause 64 bits overflow, if the + * suspended time is too long. In that case we need do the + * 64 bits math carefully + */ + do_div(max, mult); + if (cycle_delta > max) { + num = div64_u64(cycle_delta, max); + nsec = (((u64) max * mult) >> shift) * num; + cycle_delta -= num * max; + } + nsec += ((u64) cycle_delta * mult) >> shift; + + ts_delta = ns_to_timespec(nsec); + suspendtime_found = true; + } else if (timespec_compare(&ts_new, &timekeeping_suspend_time) > 0) { + ts_delta = timespec_sub(ts_new, timekeeping_suspend_time); + suspendtime_found = true; } - /* re-base the last cycle value */ - tk->clock->cycle_last = tk->clock->read(tk->clock); + + if (suspendtime_found) + __timekeeping_inject_sleeptime(tk, &ts_delta); + + /* Re-base the last cycle value */ + clock->cycle_last = cycle_now; tk->ntp_error = 0; timekeeping_suspended = 0; timekeeping_update(tk, false); -- cgit v1.2.3 From 19919226c3f20e6bf5de3df96432ce80ffd63ff2 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 22 Mar 2013 10:48:33 +0100 Subject: clockevents: Add missing tick_check_broadcast_expired() for CLOCKEVENTS=n Fengs build robot reports: arch/arm/kernel/process.c: In function 'cpu_idle': arch/arm/kernel/process.c:211:4: error: implicit declaration of function 'tick_check_broadcast_expired' [-Werror=implicit-function-declaration] Add the missing inline function for non clockevent builds Reported-by: Wu Fengguang Signed-off-by: Thomas Gleixner --- include/linux/clockchips.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/linux/clockchips.h b/include/linux/clockchips.h index 646aac136eed..464e229e7d84 100644 --- a/include/linux/clockchips.h +++ b/include/linux/clockchips.h @@ -193,6 +193,7 @@ static inline void clockevents_suspend(void) {} static inline void clockevents_resume(void) {} #define clockevents_notify(reason, arg) do { } while (0) +static inline int tick_check_broadcast_expired(void) { return 0; } #endif -- cgit v1.2.3 From cc244ddae6d4c6902ac9d7d64023534f8c44a7eb Mon Sep 17 00:00:00 2001 From: John Stultz Date: Thu, 3 May 2012 12:30:07 -0700 Subject: timekeeping: Move TAI managment into timekeeping core from ntp Currently NTP manages the TAI offset. Since there's plans for a CLOCK_TAI clockid, push the TAI management into the timekeeping core. CC: Thomas Gleixner CC: Eric Dumazet CC: Richard Cochran Signed-off-by: John Stultz --- include/linux/time.h | 2 ++ include/linux/timekeeper_internal.h | 3 +++ kernel/time/ntp.c | 18 ++++++++------- kernel/time/timekeeping.c | 44 +++++++++++++++++++++++++++++++++++++ 4 files changed, 59 insertions(+), 8 deletions(-) diff --git a/include/linux/time.h b/include/linux/time.h index d4835dfdf25e..47210a175e78 100644 --- a/include/linux/time.h +++ b/include/linux/time.h @@ -181,6 +181,8 @@ extern struct timespec timespec_trunc(struct timespec t, unsigned gran); extern int timekeeping_valid_for_hres(void); extern u64 timekeeping_max_deferment(void); extern int timekeeping_inject_offset(struct timespec *ts); +extern s32 timekeeping_get_tai_offset(void); +extern void timekeeping_set_tai_offset(s32 tai_offset); struct tms; extern void do_sys_times(struct tms *); diff --git a/include/linux/timekeeper_internal.h b/include/linux/timekeeper_internal.h index e1d558e237ec..ff94f436f8b7 100644 --- a/include/linux/timekeeper_internal.h +++ b/include/linux/timekeeper_internal.h @@ -62,6 +62,9 @@ struct timekeeper { ktime_t offs_boot; /* The raw monotonic time for the CLOCK_MONOTONIC_RAW posix clock. */ struct timespec raw_time; + /* The current UTC to TAI offset in seconds */ + s32 tai_offset; + /* Seqlock for all timekeeper values */ seqlock_t lock; }; diff --git a/kernel/time/ntp.c b/kernel/time/ntp.c index 072bb066bb7d..59e2749be0fa 100644 --- a/kernel/time/ntp.c +++ b/kernel/time/ntp.c @@ -53,9 +53,6 @@ static int time_state = TIME_OK; /* clock status bits: */ static int time_status = STA_UNSYNC; -/* TAI offset (secs): */ -static long time_tai; - /* time adjustment (nsecs): */ static s64 time_offset; @@ -415,7 +412,6 @@ int second_overflow(unsigned long secs) else if (secs % 86400 == 0) { leap = -1; time_state = TIME_OOP; - time_tai++; printk(KERN_NOTICE "Clock: inserting leap second 23:59:60 UTC\n"); } @@ -425,7 +421,6 @@ int second_overflow(unsigned long secs) time_state = TIME_OK; else if ((secs + 1) % 86400 == 0) { leap = 1; - time_tai--; time_state = TIME_WAIT; printk(KERN_NOTICE "Clock: deleting leap second 23:59:59 UTC\n"); @@ -579,7 +574,9 @@ static inline void process_adj_status(struct timex *txc, struct timespec *ts) * Called with ntp_lock held, so we can access and modify * all the global NTP state: */ -static inline void process_adjtimex_modes(struct timex *txc, struct timespec *ts) +static inline void process_adjtimex_modes(struct timex *txc, + struct timespec *ts, + s32 *time_tai) { if (txc->modes & ADJ_STATUS) process_adj_status(txc, ts); @@ -613,7 +610,7 @@ static inline void process_adjtimex_modes(struct timex *txc, struct timespec *ts } if (txc->modes & ADJ_TAI && txc->constant > 0) - time_tai = txc->constant; + *time_tai = txc->constant; if (txc->modes & ADJ_OFFSET) ntp_update_offset(txc->offset); @@ -632,6 +629,7 @@ static inline void process_adjtimex_modes(struct timex *txc, struct timespec *ts int do_adjtimex(struct timex *txc) { struct timespec ts; + u32 time_tai, orig_tai; int result; /* Validate the data before disabling interrupts */ @@ -671,6 +669,7 @@ int do_adjtimex(struct timex *txc) } getnstimeofday(&ts); + orig_tai = time_tai = timekeeping_get_tai_offset(); raw_spin_lock_irq(&ntp_lock); @@ -687,7 +686,7 @@ int do_adjtimex(struct timex *txc) /* If there are input parameters, then process them: */ if (txc->modes) - process_adjtimex_modes(txc, &ts); + process_adjtimex_modes(txc, &ts, &time_tai); txc->offset = shift_right(time_offset * NTP_INTERVAL_FREQ, NTP_SCALE_SHIFT); @@ -716,6 +715,9 @@ int do_adjtimex(struct timex *txc) raw_spin_unlock_irq(&ntp_lock); + if (time_tai != orig_tai) + timekeeping_set_tai_offset(time_tai); + txc->time.tv_sec = ts.tv_sec; txc->time.tv_usec = ts.tv_nsec; if (!(time_status & STA_NANO)) diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 0355f125d585..937098aab498 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -513,6 +513,48 @@ error: /* even if we error out, we forwarded the time, so call update */ } EXPORT_SYMBOL(timekeeping_inject_offset); + +/** + * timekeeping_get_tai_offset - Returns current TAI offset from UTC + * + */ +s32 timekeeping_get_tai_offset(void) +{ + struct timekeeper *tk = &timekeeper; + unsigned int seq; + s32 ret; + + do { + seq = read_seqbegin(&tk->lock); + ret = tk->tai_offset; + } while (read_seqretry(&tk->lock, seq)); + + return ret; +} + +/** + * __timekeeping_set_tai_offset - Lock free worker function + * + */ +void __timekeeping_set_tai_offset(struct timekeeper *tk, s32 tai_offset) +{ + tk->tai_offset = tai_offset; +} + +/** + * timekeeping_set_tai_offset - Sets the current TAI offset from UTC + * + */ +void timekeeping_set_tai_offset(s32 tai_offset) +{ + struct timekeeper *tk = &timekeeper; + unsigned long flags; + + write_seqlock_irqsave(&tk->lock, flags); + __timekeeping_set_tai_offset(tk, tai_offset); + write_sequnlock_irqrestore(&tk->lock, flags); +} + /** * change_clocksource - Swaps clocksources if a new one is available * @@ -1143,6 +1185,8 @@ static inline void accumulate_nsecs_to_secs(struct timekeeper *tk) tk_set_wall_to_mono(tk, timespec_sub(tk->wall_to_monotonic, ts)); + __timekeeping_set_tai_offset(tk, tk->tai_offset - leap); + clock_was_set_delayed(); } } -- cgit v1.2.3 From 1ff3c9677bff7e468e0c487d0ffefe4e901d33f4 Mon Sep 17 00:00:00 2001 From: John Stultz Date: Thu, 3 May 2012 12:43:40 -0700 Subject: timekeeping: Add CLOCK_TAI clockid This add a CLOCK_TAI clockid and the needed accessors. CC: Thomas Gleixner CC: Eric Dumazet CC: Richard Cochran Signed-off-by: John Stultz --- include/linux/time.h | 1 + include/uapi/linux/time.h | 6 ++---- kernel/posix-timers.c | 10 ++++++++++ kernel/time/timekeeping.c | 30 ++++++++++++++++++++++++++++++ 4 files changed, 43 insertions(+), 4 deletions(-) diff --git a/include/linux/time.h b/include/linux/time.h index 47210a175e78..22d81b3c955b 100644 --- a/include/linux/time.h +++ b/include/linux/time.h @@ -183,6 +183,7 @@ extern u64 timekeeping_max_deferment(void); extern int timekeeping_inject_offset(struct timespec *ts); extern s32 timekeeping_get_tai_offset(void); extern void timekeeping_set_tai_offset(s32 tai_offset); +extern void timekeeping_clocktai(struct timespec *ts); struct tms; extern void do_sys_times(struct tms *); diff --git a/include/uapi/linux/time.h b/include/uapi/linux/time.h index 0d3c0edc3eda..e75e1b6ff27f 100644 --- a/include/uapi/linux/time.h +++ b/include/uapi/linux/time.h @@ -54,11 +54,9 @@ struct itimerval { #define CLOCK_BOOTTIME 7 #define CLOCK_REALTIME_ALARM 8 #define CLOCK_BOOTTIME_ALARM 9 +#define CLOCK_SGI_CYCLE 10 /* Hardware specific */ +#define CLOCK_TAI 11 -/* - * The IDs of various hardware clocks: - */ -#define CLOCK_SGI_CYCLE 10 #define MAX_CLOCKS 16 #define CLOCKS_MASK (CLOCK_REALTIME | CLOCK_MONOTONIC) #define CLOCKS_MONO CLOCK_MONOTONIC diff --git a/kernel/posix-timers.c b/kernel/posix-timers.c index 6edbb2c55c22..fbfc5f1b7710 100644 --- a/kernel/posix-timers.c +++ b/kernel/posix-timers.c @@ -221,6 +221,11 @@ static int posix_get_boottime(const clockid_t which_clock, struct timespec *tp) return 0; } +static int posix_get_tai(clockid_t which_clock, struct timespec *tp) +{ + timekeeping_clocktai(tp); + return 0; +} /* * Initialize everything, well, just everything in Posix clocks/timers ;) @@ -261,6 +266,10 @@ static __init int init_posix_timers(void) .clock_getres = posix_get_coarse_res, .clock_get = posix_get_monotonic_coarse, }; + struct k_clock clock_tai = { + .clock_getres = hrtimer_get_res, + .clock_get = posix_get_tai, + }