diff options
Diffstat (limited to 'kernel')
-rw-r--r-- | kernel/cpu_pm.c | 50 | ||||
-rw-r--r-- | kernel/power/hibernate.c | 29 | ||||
-rw-r--r-- | kernel/power/main.c | 64 | ||||
-rw-r--r-- | kernel/power/power.h | 5 | ||||
-rw-r--r-- | kernel/power/suspend.c | 184 | ||||
-rw-r--r-- | kernel/power/suspend_test.c | 4 | ||||
-rw-r--r-- | kernel/sched/cpufreq_schedutil.c | 98 | ||||
-rw-r--r-- | kernel/sched/deadline.c | 2 | ||||
-rw-r--r-- | kernel/sched/fair.c | 8 | ||||
-rw-r--r-- | kernel/sched/idle.c | 8 | ||||
-rw-r--r-- | kernel/sched/rt.c | 2 | ||||
-rw-r--r-- | kernel/sched/sched.h | 10 | ||||
-rw-r--r-- | kernel/time/timekeeping_debug.c | 5 |
13 files changed, 287 insertions, 182 deletions
diff --git a/kernel/cpu_pm.c b/kernel/cpu_pm.c index 009cc9a17d95..67b02e138a47 100644 --- a/kernel/cpu_pm.c +++ b/kernel/cpu_pm.c @@ -22,15 +22,21 @@ #include <linux/spinlock.h> #include <linux/syscore_ops.h> -static DEFINE_RWLOCK(cpu_pm_notifier_lock); -static RAW_NOTIFIER_HEAD(cpu_pm_notifier_chain); +static ATOMIC_NOTIFIER_HEAD(cpu_pm_notifier_chain); static int cpu_pm_notify(enum cpu_pm_event event, int nr_to_call, int *nr_calls) { int ret; - ret = __raw_notifier_call_chain(&cpu_pm_notifier_chain, event, NULL, + /* + * __atomic_notifier_call_chain has a RCU read critical section, which + * could be disfunctional in cpu idle. Copy RCU_NONIDLE code to let + * RCU know this. + */ + rcu_irq_enter_irqson(); + ret = __atomic_notifier_call_chain(&cpu_pm_notifier_chain, event, NULL, nr_to_call, nr_calls); + rcu_irq_exit_irqson(); return notifier_to_errno(ret); } @@ -47,14 +53,7 @@ static int cpu_pm_notify(enum cpu_pm_event event, int nr_to_call, int *nr_calls) */ int cpu_pm_register_notifier(struct notifier_block *nb) { - unsigned long flags; - int ret; - - write_lock_irqsave(&cpu_pm_notifier_lock, flags); - ret = raw_notifier_chain_register(&cpu_pm_notifier_chain, nb); - write_unlock_irqrestore(&cpu_pm_notifier_lock, flags); - - return ret; + return atomic_notifier_chain_register(&cpu_pm_notifier_chain, nb); } EXPORT_SYMBOL_GPL(cpu_pm_register_notifier); @@ -69,14 +68,7 @@ EXPORT_SYMBOL_GPL(cpu_pm_register_notifier); */ int cpu_pm_unregister_notifier(struct notifier_block *nb) { - unsigned long flags; - int ret; - - write_lock_irqsave(&cpu_pm_notifier_lock, flags); - ret = raw_notifier_chain_unregister(&cpu_pm_notifier_chain, nb); - write_unlock_irqrestore(&cpu_pm_notifier_lock, flags); - - return ret; + return atomic_notifier_chain_unregister(&cpu_pm_notifier_chain, nb); } EXPORT_SYMBOL_GPL(cpu_pm_unregister_notifier); @@ -100,7 +92,6 @@ int cpu_pm_enter(void) int nr_calls; int ret = 0; - read_lock(&cpu_pm_notifier_lock); ret = cpu_pm_notify(CPU_PM_ENTER, -1, &nr_calls); if (ret) /* @@ -108,7 +99,6 @@ int cpu_pm_enter(void) * PM entry who are notified earlier to prepare for it. */ cpu_pm_notify(CPU_PM_ENTER_FAILED, nr_calls - 1, NULL); - read_unlock(&cpu_pm_notifier_lock); return ret; } @@ -128,13 +118,7 @@ EXPORT_SYMBOL_GPL(cpu_pm_enter); */ int cpu_pm_exit(void) { - int ret; - - read_lock(&cpu_pm_notifier_lock); - ret = cpu_pm_notify(CPU_PM_EXIT, -1, NULL); - read_unlock(&cpu_pm_notifier_lock); - - return ret; + return cpu_pm_notify(CPU_PM_EXIT, -1, NULL); } EXPORT_SYMBOL_GPL(cpu_pm_exit); @@ -159,7 +143,6 @@ int cpu_cluster_pm_enter(void) int nr_calls; int ret = 0; - read_lock(&cpu_pm_notifier_lock); ret = cpu_pm_notify(CPU_CLUSTER_PM_ENTER, -1, &nr_calls); if (ret) /* @@ -167,7 +150,6 @@ int cpu_cluster_pm_enter(void) * PM entry who are notified earlier to prepare for it. */ cpu_pm_notify(CPU_CLUSTER_PM_ENTER_FAILED, nr_calls - 1, NULL); - read_unlock(&cpu_pm_notifier_lock); return ret; } @@ -190,13 +172,7 @@ EXPORT_SYMBOL_GPL(cpu_cluster_pm_enter); */ int cpu_cluster_pm_exit(void) { - int ret; - - read_lock(&cpu_pm_notifier_lock); - ret = cpu_pm_notify(CPU_CLUSTER_PM_EXIT, -1, NULL); - read_unlock(&cpu_pm_notifier_lock); - - return ret; + return cpu_pm_notify(CPU_CLUSTER_PM_EXIT, -1, NULL); } EXPORT_SYMBOL_GPL(cpu_cluster_pm_exit); diff --git a/kernel/power/hibernate.c b/kernel/power/hibernate.c index e1914c7b85b1..a5c36e9c56a6 100644 --- a/kernel/power/hibernate.c +++ b/kernel/power/hibernate.c @@ -651,7 +651,7 @@ static int load_image_and_restore(void) int error; unsigned int flags; - pr_debug("Loading hibernation image.\n"); + pm_pr_dbg("Loading hibernation image.\n"); lock_device_hotplug(); error = create_basic_memory_bitmaps(); @@ -681,7 +681,7 @@ int hibernate(void) bool snapshot_test = false; if (!hibernation_available()) { - pr_debug("Hibernation not available.\n"); + pm_pr_dbg("Hibernation not available.\n"); return -EPERM; } @@ -692,6 +692,7 @@ int hibernate(void) goto Unlock; } + pr_info("hibernation entry\n"); pm_prepare_console(); error = __pm_notifier_call_chain(PM_HIBERNATION_PREPARE, -1, &nr_calls); if (error) { @@ -727,7 +728,7 @@ int hibernate(void) else flags |= SF_CRC32_MODE; - pr_debug("Writing image.\n"); + pm_pr_dbg("Writing image.\n"); error = swsusp_write(flags); swsusp_free(); if (!error) { @@ -739,7 +740,7 @@ int hibernate(void) in_suspend = 0; pm_restore_gfp_mask(); } else { - pr_debug("Image restored successfully.\n"); + pm_pr_dbg("Image restored successfully.\n"); } Free_bitmaps: @@ -747,7 +748,7 @@ int hibernate(void) Thaw: unlock_device_hotplug(); if (snapshot_test) { - pr_debug("Checking hibernation image\n"); + pm_pr_dbg("Checking hibernation image\n"); error = swsusp_check(); if (!error) error = load_image_and_restore(); @@ -762,6 +763,8 @@ int hibernate(void) atomic_inc(&snapshot_device_available); Unlock: unlock_system_sleep(); + pr_info("hibernation exit\n"); + return error; } @@ -811,7 +814,7 @@ static int software_resume(void) goto Unlock; } - pr_debug("Checking hibernation image partition %s\n", resume_file); + pm_pr_dbg("Checking hibernation image partition %s\n", resume_file); if (resume_delay) { pr_info("Waiting %dsec before reading resume device ...\n", @@ -853,10 +856,10 @@ static int software_resume(void) } Check_image: - pr_debug("Hibernation image partition %d:%d present\n", + pm_pr_dbg("Hibernation image partition %d:%d present\n", MAJOR(swsusp_resume_device), MINOR(swsusp_resume_device)); - pr_debug("Looking for hibernation image.\n"); + pm_pr_dbg("Looking for hibernation image.\n"); error = swsusp_check(); if (error) goto Unlock; @@ -868,6 +871,7 @@ static int software_resume(void) goto Unlock; } + pr_info("resume from hibernation\n"); pm_prepare_console(); error = __pm_notifier_call_chain(PM_RESTORE_PREPARE, -1, &nr_calls); if (error) { @@ -875,7 +879,7 @@ static int software_resume(void) goto Close_Finish; } - pr_debug("Preparing processes for restore.\n"); + pm_pr_dbg("Preparing processes for restore.\n"); error = freeze_processes(); if (error) goto Close_Finish; @@ -884,11 +888,12 @@ static int software_resume(void) Finish: __pm_notifier_call_chain(PM_POST_RESTORE, nr_calls, NULL); pm_restore_console(); + pr_info("resume from hibernation failed (%d)\n", error); atomic_inc(&snapshot_device_available); /* For success case, the suspend path will release the lock */ Unlock: mutex_unlock(&pm_mutex); - pr_debug("Hibernation image not present or could not be loaded.\n"); + pm_pr_dbg("Hibernation image not present or could not be loaded.\n"); return error; Close_Finish: swsusp_close(FMODE_READ); @@ -1012,8 +1017,8 @@ static ssize_t disk_store(struct kobject *kobj, struct kobj_attribute *attr, error = -EINVAL; if (!error) - pr_debug("Hibernation mode set to '%s'\n", - hibernation_modes[mode]); + pm_pr_dbg("Hibernation mode set to '%s'\n", + hibernation_modes[mode]); unlock_system_sleep(); return error ? error : n; } diff --git a/kernel/power/main.c b/kernel/power/main.c index 42bd800a6755..3a2ca9066583 100644 --- a/kernel/power/main.c +++ b/kernel/power/main.c @@ -150,7 +150,7 @@ static ssize_t mem_sleep_store(struct kobject *kobj, struct kobj_attribute *attr power_attr(mem_sleep); #endif /* CONFIG_SUSPEND */ -#ifdef CONFIG_PM_DEBUG +#ifdef CONFIG_PM_SLEEP_DEBUG int pm_test_level = TEST_NONE; static const char * const pm_tests[__TEST_AFTER_LAST] = { @@ -211,7 +211,7 @@ static ssize_t pm_test_store(struct kobject *kobj, struct kobj_attribute *attr, } power_attr(pm_test); -#endif /* CONFIG_PM_DEBUG */ +#endif /* CONFIG_PM_SLEEP_DEBUG */ #ifdef CONFIG_DEBUG_FS static char *suspend_step_name(enum suspend_stat_step step) @@ -361,6 +361,61 @@ static ssize_t pm_wakeup_irq_show(struct kobject *kobj, power_attr_ro(pm_wakeup_irq); +bool pm_debug_messages_on __read_mostly; + +static ssize_t pm_debug_messages_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return sprintf(buf, "%d\n", pm_debug_messages_on); +} + +static ssize_t pm_debug_messages_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t n) +{ + unsigned long val; + + if (kstrtoul(buf, 10, &val)) + return -EINVAL; + + if (val > 1) + return -EINVAL; + + pm_debug_messages_on = !!val; + return n; +} + +power_attr(pm_debug_messages); + +/** + * __pm_pr_dbg - Print a suspend debug message to the kernel log. + * @defer: Whether or not to use printk_deferred() to print the message. + * @fmt: Message format. + * + * The message will be emitted if enabled through the pm_debug_messages + * sysfs attribute. + */ +void __pm_pr_dbg(bool defer, const char *fmt, ...) +{ + struct va_format vaf; + va_list args; + + if (!pm_debug_messages_on) + return; + + va_start(args, fmt); + + vaf.fmt = fmt; + vaf.va = &args; + + if (defer) + printk_deferred(KERN_DEBUG "PM: %pV", &vaf); + else + printk(KERN_DEBUG "PM: %pV", &vaf); + + va_end(args); +} + #else /* !CONFIG_PM_SLEEP_DEBUG */ static inline void pm_print_times_init(void) {} #endif /* CONFIG_PM_SLEEP_DEBUG */ @@ -691,12 +746,11 @@ static struct attribute * g[] = { &wake_lock_attr.attr, &wake_unlock_attr.attr, #endif -#ifdef CONFIG_PM_DEBUG - &pm_test_attr.attr, -#endif #ifdef CONFIG_PM_SLEEP_DEBUG + &pm_test_attr.attr, &pm_print_times_attr.attr, &pm_wakeup_irq_attr.attr, + &pm_debug_messages_attr.attr, #endif #endif #ifdef CONFIG_FREEZER diff --git a/kernel/power/power.h b/kernel/power/power.h index 7fdc40d31b7d..1d2d761e3c25 100644 --- a/kernel/power/power.h +++ b/kernel/power/power.h @@ -192,7 +192,6 @@ extern void swsusp_show_speed(ktime_t, ktime_t, unsigned int, char *); extern const char * const pm_labels[]; extern const char *pm_states[]; extern const char *mem_sleep_states[]; -extern suspend_state_t mem_sleep_current; extern int suspend_devices_and_enter(suspend_state_t state); #else /* !CONFIG_SUSPEND */ @@ -245,7 +244,11 @@ enum { #define TEST_FIRST TEST_NONE #define TEST_MAX (__TEST_AFTER_LAST - 1) +#ifdef CONFIG_PM_SLEEP_DEBUG extern int pm_test_level; +#else +#define pm_test_level (TEST_NONE) +#endif #ifdef CONFIG_SUSPEND_FREEZER static inline int suspend_freeze_processes(void) diff --git a/kernel/power/suspend.c b/kernel/power/suspend.c index 3ecf275d7e44..3e2b4f519009 100644 --- a/kernel/power/suspend.c +++ b/kernel/power/suspend.c @@ -8,6 +8,8 @@ * This file is released under the GPLv2. */ +#define pr_fmt(fmt) "PM: " fmt + #include <linux/string.h> #include <linux/delay.h> #include <linux/errno.h> @@ -33,53 +35,55 @@ #include "power.h" const char * const pm_labels[] = { - [PM_SUSPEND_FREEZE] = "freeze", + [PM_SUSPEND_TO_IDLE] = "freeze", [PM_SUSPEND_STANDBY] = "standby", [PM_SUSPEND_MEM] = "mem", }; const char *pm_states[PM_SUSPEND_MAX]; static const char * const mem_sleep_labels[] = { - [PM_SUSPEND_FREEZE] = "s2idle", + [PM_SUSPEND_TO_IDLE] = "s2idle", [PM_SUSPEND_STANDBY] = "shallow", [PM_SUSPEND_MEM] = "deep", }; const char *mem_sleep_states[PM_SUSPEND_MAX]; -suspend_state_t mem_sleep_current = PM_SUSPEND_FREEZE; -static suspend_state_t mem_sleep_default = PM_SUSPEND_MEM; +suspend_state_t mem_sleep_current = PM_SUSPEND_TO_IDLE; +suspend_state_t mem_sleep_default = PM_SUSPEND_MAX; +suspend_state_t pm_suspend_target_state; +EXPORT_SYMBOL_GPL(pm_suspend_target_state); unsigned int pm_suspend_global_flags; EXPORT_SYMBOL_GPL(pm_suspend_global_flags); static const struct platform_suspend_ops *suspend_ops; -static const struct platform_freeze_ops *freeze_ops; -static DECLARE_WAIT_QUEUE_HEAD(suspend_freeze_wait_head); +static const struct platform_s2idle_ops *s2idle_ops; +static DECLARE_WAIT_QUEUE_HEAD(s2idle_wait_head); -enum freeze_state __read_mostly suspend_freeze_state; -static DEFINE_SPINLOCK(suspend_freeze_lock); +enum s2idle_states __read_mostly s2idle_state; +static DEFINE_SPINLOCK(s2idle_lock); -void freeze_set_ops(const struct platform_freeze_ops *ops) +void s2idle_set_ops(const struct platform_s2idle_ops *ops) { lock_system_sleep(); - freeze_ops = ops; + s2idle_ops = ops; unlock_system_sleep(); } -static void freeze_begin(void) +static void s2idle_begin(void) { - suspend_freeze_state = FREEZE_STATE_NONE; + s2idle_state = S2IDLE_STATE_NONE; } -static void freeze_enter(void) +static void s2idle_enter(void) { - trace_suspend_resume(TPS("machine_suspend"), PM_SUSPEND_FREEZE, true); + trace_suspend_resume(TPS("machine_suspend"), PM_SUSPEND_TO_IDLE, true); - spin_lock_irq(&suspend_freeze_lock); + spin_lock_irq(&s2idle_lock); if (pm_wakeup_pending()) goto out; - suspend_freeze_state = FREEZE_STATE_ENTER; - spin_unlock_irq(&suspend_freeze_lock); + s2idle_state = S2IDLE_STATE_ENTER; + spin_unlock_irq(&s2idle_lock); get_online_cpus(); cpuidle_resume(); @@ -87,56 +91,75 @@ static void freeze_enter(void) /* Push all the CPUs into the idle loop. */ wake_up_all_idle_cpus(); /* Make the current CPU wait so it can enter the idle loop too. */ - wait_event(suspend_freeze_wait_head, - suspend_freeze_state == FREEZE_STATE_WAKE); + wait_event(s2idle_wait_head, + s2idle_state == S2IDLE_STATE_WAKE); cpuidle_pause(); put_online_cpus(); - spin_lock_irq(&suspend_freeze_lock); + spin_lock_irq(&s2idle_lock); out: - suspend_freeze_state = FREEZE_STATE_NONE; - spin_unlock_irq(&suspend_freeze_lock); + s2idle_state = S2IDLE_STATE_NONE; + spin_unlock_irq(&s2idle_lock); - trace_suspend_resume(TPS("machine_suspend"), PM_SUSPEND_FREEZE, false); + trace_suspend_resume(TPS("machine_suspend"), PM_SUSPEND_TO_IDLE, false); } static void s2idle_loop(void) { - pr_debug("PM: suspend-to-idle\n"); + pm_pr_dbg("suspend-to-idle\n"); + + for (;;) { + int error; + + dpm_noirq_begin(); + + /* + * Suspend-to-idle equals + * frozen processes + suspended devices + idle processors. + * Thus s2idle_enter() should be called right after + * all devices have been suspended. + */ + error = dpm_noirq_suspend_devices(PMSG_SUSPEND); + if (!error) + s2idle_enter(); + + dpm_noirq_resume_devices(PMSG_RESUME); + if (error && (error != -EBUSY || !pm_wakeup_pending())) { + dpm_noirq_end(); + break; + } - do { - freeze_enter(); + if (s2idle_ops && s2idle_ops->wake) + s2idle_ops->wake(); - if (freeze_ops && freeze_ops->wake) - freeze_ops->wake(); + dpm_noirq_end(); - dpm_resume_noirq(PMSG_RESUME); - if (freeze_ops && freeze_ops->sync) - freeze_ops->sync(); + if (s2idle_ops && s2idle_ops->sync) + s2idle_ops->sync(); if (pm_wakeup_pending()) break; pm_wakeup_clear(false); - } while (!dpm_suspend_noirq(PMSG_SUSPEND)); + } - pr_debug("PM: resume from suspend-to-idle\n"); + pm_pr_dbg("resume from suspend-to-idle\n"); } -void freeze_wake(void) +void s2idle_wake(void) { unsigned long flags; - spin_lock_irqsave(&suspend_freeze_lock, flags); - if (suspend_freeze_state > FREEZE_STATE_NONE) { - suspend_freeze_state = FREEZE_STATE_WAKE; - wake_up(&suspend_freeze_wait_head); + spin_lock_irqsave(&s2idle_lock, flags); + if (s2idle_state > S2IDLE_STATE_NONE) { + s2idle_state = S2IDLE_STATE_WAKE; + wake_up(&s2idle_wait_head); } - spin_unlock_irqrestore(&suspend_freeze_lock, flags); + spin_unlock_irqrestore(&s2idle_lock, flags); } -EXPORT_SYMBOL_GPL(freeze_wake); +EXPORT_SYMBOL_GPL(s2idle_wake); static bool valid_state(suspend_state_t state) { @@ -152,19 +175,19 @@ void __init pm_states_init(void) { /* "mem" and "freeze" are always present in /sys/power/state. */ pm_states[PM_SUSPEND_MEM] = pm_labels[PM_SUSPEND_MEM]; - pm_states[PM_SUSPEND_FREEZE] = pm_labels[PM_SUSPEND_FREEZE]; + pm_states[PM_SUSPEND_TO_IDLE] = pm_labels[PM_SUSPEND_TO_IDLE]; /* * Suspend-to-idle should be supported even without any suspend_ops, * initialize mem_sleep_states[] accordingly here. */ - mem_sleep_states[PM_SUSPEND_FREEZE] = mem_sleep_labels[PM_SUSPEND_FREEZE]; + mem_sleep_states[PM_SUSPEND_TO_IDLE] = mem_sleep_labels[PM_SUSPEND_TO_IDLE]; } static int __init mem_sleep_default_setup(char *str) { suspend_state_t state; - for (state = PM_SUSPEND_FREEZE; state <= PM_SUSPEND_MEM; state++) + for (state = PM_SUSPEND_TO_IDLE; state <= PM_SUSPEND_MEM; state++) if (mem_sleep_labels[state] && !strcmp(str, mem_sleep_labels[state])) { mem_sleep_default = state; @@ -193,7 +216,7 @@ void suspend_set_ops(const struct platform_suspend_ops *ops) } if (valid_state(PM_SUSPEND_MEM)) { mem_sleep_states[PM_SUSPEND_MEM] = mem_sleep_labels[PM_SUSPEND_MEM]; - if (mem_sleep_default == PM_SUSPEND_MEM) + if (mem_sleep_default >= PM_SUSPEND_MEM) mem_sleep_current = PM_SUSPEND_MEM; } @@ -216,49 +239,49 @@ EXPORT_SYMBOL_GPL(suspend_valid_only_mem); static bool sleep_state_supported(suspend_state_t state) { - return state == PM_SUSPEND_FREEZE || (suspend_ops && suspend_ops->enter); + return state == PM_SUSPEND_TO_IDLE || (suspend_ops && suspend_ops->enter); } static int platform_suspend_prepare(suspend_state_t state) { - return state != PM_SUSPEND_FREEZE && suspend_ops->prepare ? + return state != PM_SUSPEND_TO_IDLE && suspend_ops->prepare ? suspend_ops->prepare() : 0; } static int platform_suspend_prepare_late(suspend_state_t state) { - return state == PM_SUSPEND_FREEZE && freeze_ops && freeze_ops->prepare ? - freeze_ops->prepare() : 0; + return state == PM_SUSPEND_TO_IDLE && s2idle_ops && s2idle_ops->prepare ? + s2idle_ops->prepare() : 0; } static int platform_suspend_prepare_noirq(suspend_state_t state) { - return state != PM_SUSPEND_FREEZE && suspend_ops->prepare_late ? + return state != PM_SUSPEND_TO_IDLE && suspend_ops->prepare_late ? suspend_ops->prepare_late() : 0; } static void platform_resume_noirq(suspend_state_t state) { - if (state != PM_SUSPEND_FREEZE && suspend_ops->wake) + if (state != PM_SUSPEND_TO_IDLE && suspend_ops->wake) suspend_ops->wake(); } static void platform_resume_early(suspend_state_t state) { - if (state == PM_SUSPEND_FREEZE && freeze_ops && freeze_ops->restore) - freeze_ops->restore(); + if (state == PM_SUSPEND_TO_IDLE && s2idle_ops && s2idle_ops->restore) + s2idle_ops->restore(); } static void platform_resume_finish(suspend_state_t state) { - if (state != PM_SUSPEND_FREEZE && suspend_ops->finish) + if (state != PM_SUSPEND_TO_IDLE && suspend_ops->finish) suspend_ops->finish(); } static int platform_suspend_begin(suspend_state_t state) { - if (state == PM_SUSPEND_FREEZE && freeze_ops && freeze_ops->begin) - return freeze_ops->begin(); + if (state == PM_SUSPEND_TO_IDLE && s2idle_ops && s2idle_ops->begin) + return s2idle_ops->begin(); else if (suspend_ops && suspend_ops->begin) return suspend_ops->begin(state); else @@ -267,21 +290,21 @@ static int platform_suspend_begin(suspend_state_t state) static void platform_resume_end(suspend_state_t state) { - if (state == PM_SUSPEND_FREEZE && freeze_ops && freeze_ops->end) - freeze_ops->end(); + if (state == PM_SUSPEND_TO_IDLE && s2idle_ops && s2idle_ops->end) + s2idle_ops->end(); else if (suspend_ops && suspend_ops->end) suspend_ops->end(); } static void platform_recover(suspend_state_t state) { - if (state != PM_SUSPEND_FREEZE && suspend_ops->recover) + if (state != PM_SUSPEND_TO_IDLE && suspend_ops->recover) suspend_ops->recover(); } static bool platform_suspend_again(suspend_state_t state) { - return state != PM_SUSPEND_FREEZE && suspend_ops->suspend_again ? + return state != PM_SUSPEND_TO_IDLE && suspend_ops->suspend_again ? suspend_ops->suspend_again() : false; } @@ -370,16 +393,21 @@ static int suspend_enter(suspend_state_t state, bool *wakeup) error = dpm_suspend_late(PMSG_SUSPEND); if (error) { - pr_err("PM: late suspend of devices failed\n"); + pr_err("late suspend of devices failed\n"); goto Platform_finish; } error = platform_suspend_prepare_late(state); if (error) goto Devices_early_resume; + if (state == PM_SUSPEND_TO_IDLE && pm_test_level != TEST_PLATFORM) { + s2idle_loop(); + goto Platform_early_resume; + } + error = dpm_suspend_noirq(PMSG_SUSPEND); if (error) { - pr_err("PM: noirq suspend of devices failed\n"); + pr_err("noirq suspend of devices failed\n"); goto Platform_early_resume; } error = platform_suspend_prepare_noirq(state); @@ -389,17 +417,6 @@ static int suspend_enter(suspend_state_t state, bool *wakeup) if (suspend_test(TEST_PLATFORM)) goto Platform_wake; - /* - * PM_SUSPEND_FREEZE equals - * frozen processes + suspended devices + idle processors. - * Thus we should invoke freeze_enter() soon after - * all the devices are suspended. - */ - if (state == PM_SUSPEND_FREEZE) { - s2idle_loop(); - goto Platform_early_resume; - } - error = disable_nonboot_cpus(); if (error || suspend_test(TEST_CPUS)) goto Enable_cpus; @@ -456,6 +473,8 @@ int suspend_devices_and_enter(suspend_state_t state) if (!sleep_state_supported(state)) return -ENOSYS; + pm_suspend_target_state = state; + error = platform_suspend_begin(state); if (error) goto Close; @@ -464,7 +483,7 @@ int suspend_devices_and_enter(suspend_state_t state) suspend_test_start(); error = dpm_suspend_start(PMSG_SUSPEND); if (error) { - pr_err("PM: Some devices failed to suspend, or early wake event detected\n"); + pr_err("Some devices failed to suspend, or early wake event detected\n"); goto Recover_platform; } suspend_test_finish("suspend devices"); @@ -485,6 +504,7 @@ int suspend_devices_and_enter(suspend_state_t state) Close: platform_resume_end(state); + pm_suspend_target_state = PM_SUSPEND_ON; return error; Recover_platform: @@ -518,10 +538,10 @@ static int enter_state(suspend_state_t state) int error; trace_suspend_resume(TPS("suspend_enter"), state, true); - if (state == PM_SUSPEND_FREEZE) { + if (state == PM_SUSPEND_TO_IDLE) { #ifdef CONFIG_PM_DEBUG if (pm_test_level != TEST_NONE && pm_test_level <= TEST_CPUS) { - pr_warn("PM: Unsupported test mode for suspend to idle, please choose none/freezer/devices/platform.\n"); + pr_warn("Unsupported test mode for suspend to idle, please choose none/freezer/devices/platform.\n"); return -EAGAIN; } #endif @@ -531,18 +551,18 @@ static int enter_state(suspend_state_t state) if (!mutex_trylock(&pm_mutex)) return -EBUSY; - if (state == PM_SUSPEND_FREEZE) - freeze_begin(); + if (state == PM_SUSPEND_TO_IDLE) + s2idle_begin(); #ifndef CONFIG_SUSPEND_SKIP_SYNC trace_suspend_resume(TPS("sync_filesystems"), 0, true); - pr_info("PM: Syncing filesystems ... "); + pr_info("Syncing filesystems ... "); sys_sync(); pr_cont("done.\n"); trace_suspend_resume(TPS("sync_filesystems"), 0, false); #endif - pr_debug("PM: Preparing system for sleep (%s)\n", pm_states[state]); + pm_pr_dbg("Preparing system for sleep (%s)\n", mem_sleep_labels[state]); pm_suspend_clear_flags(); error = suspend_prepare(state); if (error) @@ -552,13 +572,13 @@ static int enter_state(suspend_state_t state) goto Finish; trace_suspend_resume(TPS("suspend_enter"), state, false); - pr_debug("PM: Suspending system (%s)\n", pm_states[state]); + pm_pr_dbg("Suspending system (%s)\n", mem_sleep_labels[state]); pm_restrict_gfp_mask(); error = suspend_devices_and_enter(state); pm_restore_gfp_mask(); Finish: - pr_debug("PM: Finishing wakeup.\n"); + pm_pr_dbg("Finishing wakeup.\n"); suspend_finish(); Unlock: mutex_unlock(&pm_mutex); @@ -579,6 +599,7 @@ int pm_suspend(suspend_state_t state) if (state <= PM_SUSPEND_ON || state >= PM_SUSPEND_MAX) return -EINVAL; + pr_info("suspend entry (%s)\n", mem_sleep_labels[state]); error = enter_state(state); if (error) { suspend_stats.fail++; @@ -586,6 +607,7 @@ int pm_suspend(suspend_state_t state) } else { suspend_stats.success++; } + pr_info("suspend exit\n"); return error; } EXPORT_SYMBOL(pm_suspend); diff --git a/kernel/power/suspend_test.c b/kernel/power/suspend_test.c index 5db217051232..6a897e8b2a88 100644 --- a/kernel/power/suspend_test.c +++ b/kernel/power/suspend_test.c @@ -104,9 +104,9 @@ repeat: printk(info_test, pm_states[state]); status = pm_suspend(state); if (status < 0) - state = PM_SUSPEND_FREEZE; + state = PM_SUSPEND_TO_IDLE; } - if (state == PM_SUSPEND_FREEZE) { + if (state == PM_SUSPEND_TO_IDLE) { printk(info_test, pm_states[state]); status = pm_suspend(state); } diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c index 29a397067ffa..9209d83ecdcf 100644 --- a/kernel/sched/cpufreq_schedutil.c +++ b/kernel/sched/cpufreq_schedutil.c @@ -52,9 +52,11 @@ struct sugov_policy { struct sugov_cpu { struct update_util_data update_util; struct sugov_policy *sg_policy; + unsigned int cpu; - unsigned long iowait_boost; - unsigned long iowait_boost_max; + bool iowait_boost_pending; + unsigned int iowait_boost; + unsigned int iowait_boost_max; u64 last_update; /* The fields below are only needed when sharing a policy. */ @@ -76,6 +78,26 @@ static bool sugov_should_update_freq(struct sugov_policy *sg_policy, u64 time) { s64 delta_ns; + /* + * Since cpufreq_update_util() is called with rq->lock held for + * the @target_cpu, our per-cpu data is fully serialized. + * + * However, drivers cannot in general deal with cross-cpu + * requests, so while get_next_freq() will work, our + * sugov_update_commit() call may not for the fast switching platforms. + * + * Hence stop here for remote requests if they aren't supported + * by the hardware, as calculating the frequency is pointless if + * we cannot in fact act on it. + * + * For the slow switching platforms, the kthread is always scheduled on + * the right set of CPUs and any CPU can find the next frequency and + * schedule the kthread. + */ + if (sg_policy->policy->fast_switch_enabled && + !cpufreq_can_do_remote_dvfs(sg_policy->policy)) + return false; + if (sg_policy->work_in_progress) return false; @@ -106,7 +128,7 @@ static void sugov_update_commit(struct sugov_policy *sg_policy, u64 time, if (policy->fast_switch_enabled) { next_freq = cpufreq_driver_fast_switch(policy, next_freq); - if (next_freq == CPUFREQ_ENTRY_INVALID) + if (!next_freq) return; policy->cur = next_freq; @@ -154,12 +176,12 @@ static unsigned int get_next_freq(struct sugov_policy *sg_policy, return cpufreq_driver_resolve_freq(policy, freq); } -static void sugov_get_util(unsigned long *util, unsigned long *max) +static void sugov_get_util(unsigned long *util, unsigned long *max, int cpu) { - struct rq *rq = this_rq(); + struct rq *rq = cpu_rq(cpu); unsigned long cfs_max; - cfs_max = arch_scale_cpu_capacity(NULL, smp_processor_id()); + cfs_max = arch_scale_cpu_capacity(NULL, cpu); *util = min(rq->cfs.avg.util_avg, cfs_max); *max = cfs_max; @@ -169,30 +191,54 @@ static void sugov_set_iowait_boost(struct sugov_cpu *sg_cpu, u64 time, unsigned int flags) { if (flags & SCHED_CPUFREQ_IOWAIT) { - sg_cpu->iowait_boost = sg_cpu->iowait_boost_max; + if (sg_cpu->iowait_boost_pending) + return; + + sg_cpu->iowait_boost_pending = true; + + if (sg_cpu->iowait_boost) { + sg_cpu->iowait_boost <<= 1; + if (sg_cpu->iowait_boost > sg_cpu->iowait_boost_max) + sg_cpu->iowait_boost = sg_cpu->iowait_boost_max; + } else { + sg_cpu->iowait_boost = sg_cpu->sg_policy->policy->min; + } } else if (sg_cpu->iowait_boost) { s64 delta_ns = time - sg_cpu->last_update; /* Clear iowait_boost if the CPU apprears to have been idle. */ - if (delta_ns > TICK_NSEC) + if (delta_ns > TICK_NSEC) { sg_cpu->iowait_boost = 0; + sg_cpu->iowait_boost_pending = false; + } } } static void sugov_iowait_boost(struct sugov_cpu *sg_cpu, unsigned long *util, unsigned long *max) { - unsigned long boost_util = sg_cpu->iowait_boost; - unsigned long boost_max = sg_cpu->iowait_boost_max; + unsigned int boost_util, boost_max; - if (!boost_util) + if (!sg_cpu->iowait_boost) return; + if (sg_cpu->iowait_boost_pending) { + sg_cpu->iowait_boost_pending = false; + } else { + sg_cpu->iowait_boost >>= 1; + if (sg_cpu->iowait_boost < sg_cpu->sg_policy->policy->min) { + sg_cpu->iowait_boost = 0; + return; + } + } + + boost_util = sg_cpu->iowait_boost; + boost_max = sg_cpu->iowait_boost_max; + if (*util * boost_max < *max * boost_util) { *util = boost_util; *max = boost_max; } - sg_cpu->iowait_boost >>= 1; } #ifdef CONFIG_NO_HZ_COMMON @@ -229,7 +275,7 @@ static void sugov_update_single(struct update_util_data *hook, u64 time, if (flags & SCHED_CPUFREQ_RT_DL) { next_f = policy->cpuinfo.max_freq; } else { - sugov_get_util(&util, &max); + sugov_get_util(&util, &max, sg_cpu->cpu); sugov_iowait_boost(sg_cpu, &util, &max); next_f = get_next_freq(sg_policy, util, max); /* @@ -264,6 +310,7 @@ static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu, u64 time) delta_ns = time - j_sg_cpu->last_update; if (delta_ns > TICK_NSEC) { j_sg_cpu->iowait_boost = 0; + j_sg_cpu->iowait_boost_pending = false; continue; } if (j_sg_cpu->flags & SCHED_CPUFREQ_RT_DL) @@ -290,7 +337,7 @@ static void sugov_update_shared(struct update_util_data *hook, u64 time, unsigned long util, max; unsigned int next_f; - sugov_get_util(&util, &max); + sugov_get_util(&util, &max, sg_cpu->cpu); raw_spin_lock(&sg_policy->update_lock); @@ -445,7 +492,11 @@ static int sugov_kthread_create(struct sugov_policy *sg_policy) } sg_policy->thread = thread; - kthread_bind_mask(thread, policy->related_cpus); + + /* Kthread is bound to all CPUs by default */ + if (!policy->dvfs_possible_from_any_cpu) + kthread_bind_mask(thread, policy->related_cpus); + init_irq_work(&sg_policy->irq_work, sugov_irq_work); mutex_init(&sg_policy->work_lock); @@ -528,16 +579,7 @@ static int sugov_init(struct cpufreq_policy *policy) goto stop_kthread; } - if (policy->transition_delay_us) { - tunables->rate_limit_us = policy->transition_delay_us; - } else { - unsigned int lat; - - tunables->rate_limit_us = LATENCY_MULTIPLIER; - lat = policy->cpuinfo.transition_latency / NSEC_PER_USEC; - if (lat) - tunables->rate_limit_us *= lat; - } + tunables->rate_limit_us = cpufreq_policy_transition_delay_us(policy); policy->governor_data = sg_policy; sg_policy->tunables = tunables; @@ -655,6 +697,7 @@ static void sugov_limits(struct cpufreq_policy *policy) static struct cpufreq_governor schedutil_gov = { .name = "schedutil", .owner = THIS_MODULE, + .dynamic_switching = true, .init = sugov_init, .exit = sugov_exit, .start = sugov_start, @@ -671,6 +714,11 @@ struct cpufreq_governor *cpufreq_default_governor(void) static int __init sugov_register(void) { + int cpu; + + for_each_possible_cpu(cpu) |