| /* |
| * kernel/sched.c |
| * |
| * Kernel scheduler and related syscalls |
| * |
| * Copyright (C) 1991-2002 Linus Torvalds |
| * |
| * 1996-12-23 Modified by Dave Grothe to fix bugs in semaphores and |
| * make semaphores SMP safe |
| * 1998-11-19 Implemented schedule_timeout() and related stuff |
| * by Andrea Arcangeli |
| * 2002-01-04 New ultra-scalable O(1) scheduler by Ingo Molnar: |
| * hybrid priority-list and round-robin design with |
| * an array-switch method of distributing timeslices |
| * and per-CPU runqueues. Cleanups and useful suggestions |
| * by Davide Libenzi, preemptible kernel bits by Robert Love. |
| * 2003-09-03 Interactivity tuning by Con Kolivas. |
| * 2004-04-02 Scheduler domains code by Nick Piggin |
| * 2007-04-15 Work begun on replacing all interactivity tuning with a |
| * fair scheduling design by Con Kolivas. |
| * 2007-05-05 Load balancing (smp-nice) and other improvements |
| * by Peter Williams |
| * 2007-05-06 Interactivity improvements to CFS by Mike Galbraith |
| * 2007-07-01 Group scheduling enhancements by Srivatsa Vaddagiri |
| * 2007-11-29 RT balancing improvements by Steven Rostedt, Gregory Haskins, |
| * Thomas Gleixner, Mike Kravetz |
| */ |
| |
| #include <linux/mm.h> |
| #include <linux/module.h> |
| #include <linux/nmi.h> |
| #include <linux/init.h> |
| #include <linux/uaccess.h> |
| #include <linux/highmem.h> |
| #include <asm/mmu_context.h> |
| #include <linux/interrupt.h> |
| #include <linux/capability.h> |
| #include <linux/completion.h> |
| #include <linux/kernel_stat.h> |
| #include <linux/debug_locks.h> |
| #include <linux/perf_event.h> |
| #include <linux/security.h> |
| #include <linux/notifier.h> |
| #include <linux/profile.h> |
| #include <linux/freezer.h> |
| #include <linux/vmalloc.h> |
| #include <linux/blkdev.h> |
| #include <linux/delay.h> |
| #include <linux/pid_namespace.h> |
| #include <linux/smp.h> |
| #include <linux/threads.h> |
| #include <linux/timer.h> |
| #include <linux/rcupdate.h> |
| #include <linux/cpu.h> |
| #include <linux/cpuset.h> |
| #include <linux/percpu.h> |
| #include <linux/proc_fs.h> |
| #include <linux/seq_file.h> |
| #include <linux/stop_machine.h> |
| #include <linux/sysctl.h> |
| #include <linux/syscalls.h> |
| #include <linux/times.h> |
| #include <linux/tsacct_kern.h> |
| #include <linux/kprobes.h> |
| #include <linux/delayacct.h> |
| #include <linux/unistd.h> |
| #include <linux/pagemap.h> |
| #include <linux/hrtimer.h> |
| #include <linux/tick.h> |
| #include <linux/debugfs.h> |
| #include <linux/ctype.h> |
| #include <linux/ftrace.h> |
| #include <linux/slab.h> |
| #include <linux/init_task.h> |
| |
| #include <asm/tlb.h> |
| #include <asm/irq_regs.h> |
| #include <asm/mutex.h> |
| #ifdef CONFIG_PARAVIRT |
| #include <asm/paravirt.h> |
| #endif |
| |
| #include "sched_cpupri.h" |
| #include "workqueue_sched.h" |
| #include "sched_autogroup.h" |
| |
| #define CREATE_TRACE_POINTS |
| #include <trace/events/sched.h> |
| |
| /* |
| * Convert user-nice values [ -20 ... 0 ... 19 ] |
| * to static priority [ MAX_RT_PRIO..MAX_PRIO-1 ], |
| * and back. |
| */ |
| #define NICE_TO_PRIO(nice) (MAX_RT_PRIO + (nice) + 20) |
| #define PRIO_TO_NICE(prio) ((prio) - MAX_RT_PRIO - 20) |
| #define TASK_NICE(p) PRIO_TO_NICE((p)->static_prio) |
| |
| /* |
| * 'User priority' is the nice value converted to something we |
| * can work with better when scaling various scheduler parameters, |
| * it's a [ 0 ... 39 ] range. |
| */ |
| #define USER_PRIO(p) ((p)-MAX_RT_PRIO) |
| #define TASK_USER_PRIO(p) USER_PRIO((p)->static_prio) |
| #define MAX_USER_PRIO (USER_PRIO(MAX_PRIO)) |
| |
| /* |
| * Helpers for converting nanosecond timing to jiffy resolution |
| */ |
| #define NS_TO_JIFFIES(TIME) ((unsigned long)(TIME) / (NSEC_PER_SEC / HZ)) |
| |
| #define NICE_0_LOAD SCHED_LOAD_SCALE |
| #define NICE_0_SHIFT SCHED_LOAD_SHIFT |
| |
| /* |
| * These are the 'tuning knobs' of the scheduler: |
| * |
| * default timeslice is 100 msecs (used only for SCHED_RR tasks). |
| * Timeslices get refilled after they expire. |
| */ |
| #define DEF_TIMESLICE (100 * HZ / 1000) |
| |
| /* |
| * single value that denotes runtime == period, ie unlimited time. |
| */ |
| #define RUNTIME_INF ((u64)~0ULL) |
| |
| static inline int rt_policy(int policy) |
| { |
| if (policy == SCHED_FIFO || policy == SCHED_RR) |
| return 1; |
| return 0; |
| } |
| |
| static inline int task_has_rt_policy(struct task_struct *p) |
| { |
| return rt_policy(p->policy); |
| } |
| |
| /* |
| * This is the priority-queue data structure of the RT scheduling class: |
| */ |
| struct rt_prio_array { |
| DECLARE_BITMAP(bitmap, MAX_RT_PRIO+1); /* include 1 bit for delimiter */ |
| struct list_head queue[MAX_RT_PRIO]; |
| }; |
| |
| struct rt_bandwidth { |
| /* nests inside the rq lock: */ |
| raw_spinlock_t rt_runtime_lock; |
| ktime_t rt_period; |
| u64 rt_runtime; |
| struct hrtimer rt_period_timer; |
| }; |
| |
| static struct rt_bandwidth def_rt_bandwidth; |
| |
| static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun); |
| |
| static enum hrtimer_restart sched_rt_period_timer(struct hrtimer *timer) |
| { |
| struct rt_bandwidth *rt_b = |
| container_of(timer, struct rt_bandwidth, rt_period_timer); |
| ktime_t now; |
| int overrun; |
| int idle = 0; |
| |
| for (;;) { |
| now = hrtimer_cb_get_time(timer); |
| overrun = hrtimer_forward(timer, now, rt_b->rt_period); |
| |
| if (!overrun) |
| break; |
| |
| idle = do_sched_rt_period_timer(rt_b, overrun); |
| } |
| |
| return idle ? HRTIMER_NORESTART : HRTIMER_RESTART; |
| } |
| |
| static |
| void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime) |
| { |
| rt_b->rt_period = ns_to_ktime(period); |
| rt_b->rt_runtime = runtime; |
| |
| raw_spin_lock_init(&rt_b->rt_runtime_lock); |
| |
| hrtimer_init(&rt_b->rt_period_timer, |
| CLOCK_MONOTONIC, HRTIMER_MODE_REL); |
| rt_b->rt_period_timer.function = sched_rt_period_timer; |
| } |
| |
| static inline int rt_bandwidth_enabled(void) |
| { |
| return sysctl_sched_rt_runtime >= 0; |
| } |
| |
| static void start_bandwidth_timer(struct hrtimer *period_timer, ktime_t period) |
| { |
| unsigned long delta; |
| ktime_t soft, hard, now; |
| |
| for (;;) { |
| if (hrtimer_active(period_timer)) |
| break; |
| |
| now = hrtimer_cb_get_time(period_timer); |
| hrtimer_forward(period_timer, now, period); |
| |
| soft = hrtimer_get_softexpires(period_timer); |
| hard = hrtimer_get_expires(period_timer); |
| delta = ktime_to_ns(ktime_sub(hard, soft)); |
| __hrtimer_start_range_ns(period_timer, soft, delta, |
| HRTIMER_MODE_ABS_PINNED, 0); |
| } |
| } |
| |
| static void start_rt_bandwidth(struct rt_bandwidth *rt_b) |
| { |
| if (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF) |
| return; |
| |
| if (hrtimer_active(&rt_b->rt_period_timer)) |
| return; |
| |
| raw_spin_lock(&rt_b->rt_runtime_lock); |
| start_bandwidth_timer(&rt_b->rt_period_timer, rt_b->rt_period); |
| raw_spin_unlock(&rt_b->rt_runtime_lock); |
| } |
| |
| #ifdef CONFIG_RT_GROUP_SCHED |
| static void destroy_rt_bandwidth(struct rt_bandwidth *rt_b) |
| { |
| hrtimer_cancel(&rt_b->rt_period_timer); |
| } |
| #endif |
| |
| /* |
| * sched_domains_mutex serializes calls to init_sched_domains, |
| * detach_destroy_domains and partition_sched_domains. |
| */ |
| static DEFINE_MUTEX(sched_domains_mutex); |
| |
| #ifdef CONFIG_CGROUP_SCHED |
| |
| #include <linux/cgroup.h> |
| |
| struct cfs_rq; |
| |
| static LIST_HEAD(task_groups); |
| |
| struct cfs_bandwidth { |
| #ifdef CONFIG_CFS_BANDWIDTH |
| raw_spinlock_t lock; |
| ktime_t period; |
| u64 quota, runtime; |
| s64 hierarchal_quota; |
| u64 runtime_expires; |
| |
| int idle, timer_active; |
| struct hrtimer period_timer, slack_timer; |
| struct list_head throttled_cfs_rq; |
| |
| /* statistics */ |
| int nr_periods, nr_throttled; |
| u64 throttled_time; |
| #endif |
| }; |
| |
| /* task group related information */ |
| struct task_group { |
| struct cgroup_subsys_state css; |
| |
| #ifdef CONFIG_FAIR_GROUP_SCHED |
| /* schedulable entities of this group on each cpu */ |
| struct sched_entity **se; |
| /* runqueue "owned" by this group on each cpu */ |
| struct cfs_rq **cfs_rq; |
| unsigned long shares; |
| |
| atomic_t load_weight; |
| #endif |
| |
| #ifdef CONFIG_RT_GROUP_SCHED |
| struct sched_rt_entity **rt_se; |
| struct rt_rq **rt_rq; |
| |
| struct rt_bandwidth rt_bandwidth; |
| #endif |
| |
| struct rcu_head rcu; |
| struct list_head list; |
| |
| struct task_group *parent; |
| struct list_head siblings; |
| struct list_head children; |
| |
| #ifdef CONFIG_SCHED_AUTOGROUP |
| struct autogroup *autogroup; |
| #endif |
| |
| struct cfs_bandwidth cfs_bandwidth; |
| }; |
| |
| /* task_group_lock serializes the addition/removal of task groups */ |
| static DEFINE_SPINLOCK(task_group_lock); |
| |
| #ifdef CONFIG_FAIR_GROUP_SCHED |
| |
| # define ROOT_TASK_GROUP_LOAD NICE_0_LOAD |
| |
| /* |
| * A weight of 0 or 1 can cause arithmetics problems. |
| * A weight of a cfs_rq is the sum of weights of which entities |
| * are queued on this cfs_rq, so a weight of a entity should not be |
| * too large, so as the shares value of a task group. |
| * (The default weight is 1024 - so there's no practical |
| * limitation from this.) |
| */ |
| #define MIN_SHARES (1UL << 1) |
| #define MAX_SHARES (1UL << 18) |
| |
| static int root_task_group_load = ROOT_TASK_GROUP_LOAD; |
| #endif |
| |
| /* Default task group. |
| * Every task in system belong to this group at bootup. |
| */ |
| struct task_group root_task_group; |
| |
| #endif /* CONFIG_CGROUP_SCHED */ |
| |
| /* CFS-related fields in a runqueue */ |
| struct cfs_rq { |
| struct load_weight load; |
| unsigned long nr_running, h_nr_running; |
| |
| u64 exec_clock; |
| u64 min_vruntime; |
| #ifndef CONFIG_64BIT |
| u64 min_vruntime_copy; |
| #endif |
| |
| struct rb_root tasks_timeline; |
| struct rb_node *rb_leftmost; |
| |
| struct list_head tasks; |
| struct list_head *balance_iterator; |
| |
| /* |
| * 'curr' points to currently running entity on this cfs_rq. |
| * It is set to NULL otherwise (i.e when none are currently running). |
| */ |
| struct sched_entity *curr, *next, *last, *skip; |
| |
| #ifdef CONFIG_SCHED_DEBUG |
| unsigned int nr_spread_over; |
| #endif |
| |
| #ifdef CONFIG_FAIR_GROUP_SCHED |
| struct rq *rq; /* cpu runqueue to which this cfs_rq is attached */ |
| |
| /* |
| * leaf cfs_rqs are those that hold tasks (lowest schedulable entity in |
| * a hierarchy). Non-leaf lrqs hold other higher schedulable entities |
| * (like users, containers etc.) |
| * |
| * leaf_cfs_rq_list ties together list of leaf cfs_rq's in a cpu. This |
| * list is used during load balance. |
| */ |
| int on_list; |
| struct list_head leaf_cfs_rq_list; |
| struct task_group *tg; /* group that "owns" this runqueue */ |
| |
| #ifdef CONFIG_SMP |
| /* |
| * the part of load.weight contributed by tasks |
| */ |
| unsigned long task_weight; |
| |
| /* |
| * h_load = weight * f(tg) |
| * |
| * Where f(tg) is the recursive weight fraction assigned to |
| * this group. |
| */ |
| unsigned long h_load; |
| |
| /* |
| * Maintaining per-cpu shares distribution for group scheduling |
| * |
| * load_stamp is the last time we updated the load average |
| * load_last is the last time we updated the load average and saw load |
| * load_unacc_exec_time is currently unaccounted execution time |
| */ |
| u64 load_avg; |
| u64 load_period; |
| u64 load_stamp, load_last, load_unacc_exec_time; |
| |
| unsigned long load_contribution; |
| #endif |
| #ifdef CONFIG_CFS_BANDWIDTH |
| int runtime_enabled; |
| u64 runtime_expires; |
| s64 runtime_remaining; |
| |
| u64 throttled_timestamp; |
| int throttled, throttle_count; |
| struct list_head throttled_list; |
| #endif |
| #endif |
| }; |
| |
| #ifdef CONFIG_FAIR_GROUP_SCHED |
| #ifdef CONFIG_CFS_BANDWIDTH |
| static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg) |
| { |
| return &tg->cfs_bandwidth; |
| } |
| |
| static inline u64 default_cfs_period(void); |
| static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun); |
| static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b); |
| |
| static enum hrtimer_restart sched_cfs_slack_timer(struct hrtimer *timer) |
| { |
| struct cfs_bandwidth *cfs_b = |
| container_of(timer, struct cfs_bandwidth, slack_timer); |
| do_sched_cfs_slack_timer(cfs_b); |
| |
| return HRTIMER_NORESTART; |
| } |
| |
| static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer) |
| { |
| struct cfs_bandwidth *cfs_b = |
| container_of(timer, struct cfs_bandwidth, period_timer); |
| ktime_t now; |
| int overrun; |
| int idle = 0; |
| |
| for (;;) { |
| now = hrtimer_cb_get_time(timer); |
| overrun = hrtimer_forward(timer, now, cfs_b->period); |
| |
| if (!overrun) |
| break; |
| |
| idle = do_sched_cfs_period_timer(cfs_b, overrun); |
| } |
| |
| return idle ? HRTIMER_NORESTART : HRTIMER_RESTART; |
| } |
| |
| static void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) |
| { |
| raw_spin_lock_init(&cfs_b->lock); |
| cfs_b->runtime = 0; |
| cfs_b->quota = RUNTIME_INF; |
| cfs_b->period = ns_to_ktime(default_cfs_period()); |
| |
| INIT_LIST_HEAD(&cfs_b->throttled_cfs_rq); |
| hrtimer_init(&cfs_b->period_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); |
| cfs_b->period_timer.function = sched_cfs_period_timer; |
| hrtimer_init(&cfs_b->slack_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); |
| cfs_b->slack_timer.function = sched_cfs_slack_timer; |
| } |
| |
| static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) |
| { |
| cfs_rq->runtime_enabled = 0; |
| INIT_LIST_HEAD(&cfs_rq->throttled_list); |
| } |
| |
| /* requires cfs_b->lock, may release to reprogram timer */ |
| static void __start_cfs_bandwidth(struct cfs_bandwidth *cfs_b) |
| { |
| /* |
| * The timer may be active because we're trying to set a new bandwidth |
| * period or because we're racing with the tear-down path |
| * (timer_active==0 becomes visible before the hrtimer call-back |
| * terminates). In either case we ensure that it's re-programmed |
| */ |
| while (unlikely(hrtimer_active(&cfs_b->period_timer))) { |
| raw_spin_unlock(&cfs_b->lock); |
| /* ensure cfs_b->lock is available while we wait */ |
| hrtimer_cancel(&cfs_b->period_timer); |
| |
| raw_spin_lock(&cfs_b->lock); |
| /* if someone else restarted the timer then we're done */ |
| if (cfs_b->timer_active) |
| return; |
| } |
| |
| cfs_b->timer_active = 1; |
| start_bandwidth_timer(&cfs_b->period_timer, cfs_b->period); |
| } |
| |
| static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) |
| { |
| hrtimer_cancel(&cfs_b->period_timer); |
| hrtimer_cancel(&cfs_b->slack_timer); |
| } |
| #else |
| static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) {} |
| static void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {} |
| static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {} |
| |
| static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg) |
| { |
| return NULL; |
| } |
| #endif /* CONFIG_CFS_BANDWIDTH */ |
| #endif /* CONFIG_FAIR_GROUP_SCHED */ |
| |
| /* Real-Time classes' related field in a runqueue: */ |
| struct rt_rq { |
| struct rt_prio_array active; |
| unsigned long rt_nr_running; |
| #if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED |
| struct { |
| int curr; /* highest queued rt task prio */ |
| #ifdef CONFIG_SMP |
| int next; /* next highest */ |
| #endif |
| } highest_prio; |
| #endif |
| #ifdef CONFIG_SMP |
| unsigned long rt_nr_migratory; |
| unsigned long rt_nr_total; |
| int overloaded; |
| struct plist_head pushable_tasks; |
| #endif |
| int rt_throttled; |
| u64 rt_time; |
| u64 rt_runtime; |
| /* Nests inside the rq lock: */ |
| raw_spinlock_t rt_runtime_lock; |
| |
| #ifdef CONFIG_RT_GROUP_SCHED |
| unsigned long rt_nr_boosted; |
| |
| struct rq *rq; |
| struct list_head leaf_rt_rq_list; |
| struct task_group *tg; |
| #endif |
| }; |
| |
| #ifdef CONFIG_SMP |
| |
| /* |
| * We add the notion of a root-domain which will be used to define per-domain |
| * variables. Each exclusive cpuset essentially defines an island domain by |
| * fully partitioning the member cpus from any other cpuset. Whenever a new |
| * exclusive cpuset is created, we also create and attach a new root-domain |
| * object. |
| * |
| */ |
| struct root_domain { |
| atomic_t refcount; |
| atomic_t rto_count; |
| struct rcu_head rcu; |
| cpumask_var_t span; |
| cpumask_var_t online; |
| |
| /* |
| * The "RT overload" flag: it gets set if a CPU has more than |
| * one runnable RT task. |
| */ |
| cpumask_var_t rto_mask; |
| struct cpupri cpupri; |
| }; |
| |
| /* |
| * By default the system creates a single root-domain with all cpus as |
| * members (mimicking the global state we have today). |
| */ |
| static struct root_domain def_root_domain; |
| |
| #endif /* CONFIG_SMP */ |
| |
| /* |
| * This is the main, per-CPU runqueue data structure. |
| * |
| * Locking rule: those places that want to lock multiple runqueues |
| * (such as the load balancing or the thread migration code), lock |
| * acquire operations must be ordered by ascending &runqueue. |
| */ |
| struct rq { |
| /* runqueue lock: */ |
| raw_spinlock_t lock; |
| |
| /* |
| * nr_running and cpu_load should be in the same cacheline because |
| * remote CPUs use both these fields when doing load calculation. |
| */ |
| unsigned long nr_running; |
| #define CPU_LOAD_IDX_MAX 5 |
| unsigned long cpu_load[CPU_LOAD_IDX_MAX]; |
| unsigned long last_load_update_tick; |
| #ifdef CONFIG_NO_HZ |
| u64 nohz_stamp; |
| unsigned char nohz_balance_kick; |
| #endif |
| int skip_clock_update; |
| |
| /* capture load from *all* tasks on this cpu: */ |
| struct load_weight load; |
| unsigned long nr_load_updates; |
| u64 nr_switches; |
| |
| struct cfs_rq cfs; |
| struct rt_rq rt; |
| |
| #ifdef CONFIG_FAIR_GROUP_SCHED |
| /* list of leaf cfs_rq on this cpu: */ |
| struct list_head leaf_cfs_rq_list; |
| #endif |
| #ifdef CONFIG_RT_GROUP_SCHED |
| struct list_head leaf_rt_rq_list; |
| #endif |
| |
| /* |
| * This is part of a global counter where only the total sum |
| * over all CPUs matters. A task can increase this counter on |
| * one CPU and if it got migrated afterwards it may decrease |
| * it on another CPU. Always updated under the runqueue lock: |
| */ |
| unsigned long nr_uninterruptible; |
| |
| struct task_struct *curr, *idle, *stop; |
| unsigned long next_balance; |
| struct mm_struct *prev_mm; |
| |
| u64 clock; |
| u64 clock_task; |
| |
| atomic_t nr_iowait; |
| |
| #ifdef CONFIG_SMP |
| struct root_domain *rd; |
| struct sched_domain *sd; |
| |
| unsigned long cpu_power; |
| |
| unsigned char idle_balance; |
| /* For active balancing */ |
| int post_schedule; |
| int active_balance; |
| int push_cpu; |
| struct cpu_stop_work active_balance_work; |
| /* cpu of this runqueue: */ |
| int cpu; |
| int online; |
| |
| u64 rt_avg; |
| u64 age_stamp; |
| u64 idle_stamp; |
| u64 avg_idle; |
| #endif |
| |
| #ifdef CONFIG_IRQ_TIME_ACCOUNTING |
| u64 prev_irq_time; |
| #endif |
| #ifdef CONFIG_PARAVIRT |
| u64 prev_steal_time; |
| #endif |
| #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING |
| u64 prev_steal_time_rq; |
| #endif |
| |
| /* calc_load related fields */ |
| unsigned long calc_load_update; |
| long calc_load_active; |
| |
| #ifdef CONFIG_SCHED_HRTICK |
| #ifdef CONFIG_SMP |
| int hrtick_csd_pending; |
| struct call_single_data hrtick_csd; |
| #endif |
| struct hrtimer hrtick_timer; |
| #endif |
| |
| #ifdef CONFIG_SCHEDSTATS |
| /* latency stats */ |
| struct sched_info rq_sched_info; |
| unsigned long long rq_cpu_time; |
| /* could above be rq->cfs_rq.exec_clock + rq->rt_rq.rt_runtime ? */ |
| |
| /* sys_sched_yield() stats */ |
| unsigned int yld_count; |
| |
| /* schedule() stats */ |
| unsigned int sched_switch; |
| unsigned int sched_count; |
| unsigned int sched_goidle; |
| |
| /* try_to_wake_up() stats */ |
| unsigned int ttwu_count; |
| unsigned int ttwu_local; |
| #endif |
| |
| #ifdef CONFIG_SMP |
| struct llist_head wake_list; |
| #endif |
| }; |
| |
| static DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); |
| |
| |
| static void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags); |
| |
| static inline int cpu_of(struct rq *rq) |
| { |
| #ifdef CONFIG_SMP |
| return rq->cpu; |
| #else |
| return 0; |
| #endif |
| } |
| |
| #define rcu_dereference_check_sched_domain(p) \ |
| rcu_dereference_check((p), \ |
| lockdep_is_held(&sched_domains_mutex)) |
| |
| /* |
| * The domain tree (rq->sd) is protected by RCU's quiescent state transition. |
| * See detach_destroy_domains: synchronize_sched for details. |
| * |
| * The domain tree of any CPU may only be accessed from within |
| * preempt-disabled sections. |
| */ |
| #define for_each_domain(cpu, __sd) \ |
| for (__sd = rcu_dereference_check_sched_domain(cpu_rq(cpu)->sd); __sd; __sd = __sd->parent) |
| |
| #define cpu_rq(cpu) (&per_cpu(runqueues, (cpu))) |
| #define this_rq() (&__get_cpu_var(runqueues)) |
| #define task_rq(p) cpu_rq(task_cpu(p)) |
| #define cpu_curr(cpu) (cpu_rq(cpu)->curr) |
| #define raw_rq() (&__raw_get_cpu_var(runqueues)) |
| |
| #ifdef CONFIG_CGROUP_SCHED |
| |
| /* |
| * Return the group to which this tasks belongs. |
| * |
| * We use task_subsys_state_check() and extend the RCU verification with |
| * pi->lock and rq->lock because cpu_cgroup_attach() holds those locks for each |
| * task it moves into the cgroup. Therefore by holding either of those locks, |
| * we pin the task to the current cgroup. |
| */ |
| static inline struct task_group *task_group(struct task_struct *p) |
| { |
| struct task_group *tg; |
| struct cgroup_subsys_state *css; |
| |
| css = task_subsys_state_check(p, cpu_cgroup_subsys_id, |
| lockdep_is_held(&p->pi_lock) || |
| lockdep_is_held(&task_rq(p)->lock)); |
| tg = container_of(css, struct task_group, css); |
| |
| return autogroup_task_group(p, tg); |
| } |
| |
| /* Change a task's cfs_rq and parent entity if it moves across CPUs/groups */ |
| static inline void set_task_rq(struct task_struct *p, unsigned int cpu) |
| { |
| #ifdef CONFIG_FAIR_GROUP_SCHED |
| p->se.cfs_rq = task_group(p)->cfs_rq[cpu]; |
| p->se.parent = task_group(p)->se[cpu]; |
| #endif |
| |
| #ifdef CONFIG_RT_GROUP_SCHED |
| p->rt.rt_rq = task_group(p)->rt_rq[cpu]; |
| p->rt.parent = task_group(p)->rt_se[cpu]; |
| #endif |
| } |
| |
| #else /* CONFIG_CGROUP_SCHED */ |
| |
| static inline void set_task_rq(struct task_struct *p, unsigned int cpu) { } |
| static inline struct task_group *task_group(struct task_struct *p) |
| { |
| return NULL; |
| } |
| |
| #endif /* CONFIG_CGROUP_SCHED */ |
| |
| static void update_rq_clock_task(struct rq *rq, s64 delta); |
| |
| static void update_rq_clock(struct rq *rq) |
| { |
| s64 delta; |
| |
| if (rq->skip_clock_update > 0) |
| return; |
| |
| delta = sched_clock_cpu(cpu_of(rq)) - rq->clock; |
| rq->clock += delta; |
| update_rq_clock_task(rq, delta); |
| } |
| |
| /* |
| * Tunables that become constants when CONFIG_SCHED_DEBUG is off: |
| */ |
| #ifdef CONFIG_SCHED_DEBUG |
| # define const_debug __read_mostly |
| #else |
| # define const_debug static const |
| #endif |
| |
| /** |
| * runqueue_is_locked - Returns true if the current cpu runqueue is locked |
| * @cpu: the processor in question. |
| * |
| * This interface allows printk to be called with the runqueue lock |
| * held and know whether or not it is OK to wake up the klogd. |
| */ |
| int runqueue_is_locked(int cpu) |
| { |
| return raw_spin_is_locked(&cpu_rq(cpu)->lock); |
| } |
| |
| /* |
| * Debugging: various feature bits |
| */ |
| |
| #define SCHED_FEAT(name, enabled) \ |
| __SCHED_FEAT_##name , |
| |
| enum { |
| #include "sched_features.h" |
| }; |
| |
| #undef SCHED_FEAT |
| |
| #define SCHED_FEAT(name, enabled) \ |
| (1UL << __SCHED_FEAT_##name) * enabled | |
| |
| const_debug unsigned int sysctl_sched_features = |
| #include "sched_features.h" |
| 0; |
| |
| #undef SCHED_FEAT |
| |
| #ifdef CONFIG_SCHED_DEBUG |
| #define SCHED_FEAT(name, enabled) \ |
| #name , |
| |
| static __read_mostly char *sched_feat_names[] = { |
| #include "sched_features.h" |
| NULL |
| }; |
| |
| #undef SCHED_FEAT |
| |
| static int sched_feat_show(struct seq_file *m, void *v) |
| { |
| int i; |
| |
| for (i = 0; sched_feat_names[i]; i++) { |
| if (!(sysctl_sched_features & (1UL << i))) |
| seq_puts(m, "NO_"); |
| seq_printf(m, "%s ", sched_feat_names[i]); |
| } |
| seq_puts(m, "\n"); |
| |
| return 0; |
| } |
| |
| static ssize_t |
| sched_feat_write(struct file *filp, const char __user *ubuf, |
| size_t cnt, loff_t *ppos) |
| { |
| char buf[64]; |
| char *cmp; |
| int neg = 0; |
| int i; |
| |
| if (cnt > 63) |
| cnt = 63; |
| |
| if (copy_from_user(&buf, ubuf, cnt)) |
| return -EFAULT; |
| |
| buf[cnt] = 0; |
| cmp = strstrip(buf); |
| |
| if (strncmp(cmp, "NO_", 3) == 0) { |
| neg = 1; |
| cmp += 3; |
| } |
| |
| for (i = 0; sched_feat_names[i]; i++) { |
| if (strcmp(cmp, sched_feat_names[i]) == 0) { |
| if (neg) |
| sysctl_sched_features &= ~(1UL << i); |
| else |
| sysctl_sched_features |= (1UL << i); |
| break; |
| } |
| } |
| |
| if (!sched_feat_names[i]) |
| return -EINVAL; |
| |
| *ppos += cnt; |
| |
| return cnt; |
| } |
| |
| static int sched_feat_open(struct inode *inode, struct file *filp) |
| { |
| return single_open(filp, sched_feat_show, NULL); |
| } |
| |
| static const struct file_operations sched_feat_fops = { |
| .open = sched_feat_open, |
| .write = sched_feat_write, |
| .read = seq_read, |
| .llseek = seq_lseek, |
| .release = single_release, |
| }; |
| |
| static __init int sched_init_debug(void) |
| { |
| debugfs_create_file("sched_features", 0644, NULL, NULL, |
| &sched_feat_fops); |
| |
| return 0; |
| } |
| late_initcall(sched_init_debug); |
| |
| #endif |
| |
| #define sched_feat(x) (sysctl_sched_features & (1UL << __SCHED_FEAT_##x)) |
| |
| /* |
| * Number of tasks to iterate in a single balance run. |
| * Limited because this is done with IRQs disabled. |
| */ |
| const_debug unsigned int sysctl_sched_nr_migrate = 32; |
| |
| /* |
| * period over which we average the RT time consumption, measured |
| * in ms. |
| * |
| * default: 1s |
| */ |
| const_debug unsigned int sysctl_sched_time_avg = MSEC_PER_SEC; |
| |
| /* |
| * period over which we measure -rt task cpu usage in us. |
| * default: 1s |
| */ |
| unsigned int sysctl_sched_rt_period = 1000000; |
| |
| static __read_mostly int scheduler_running; |
| |
| /* |
| * part of the period that we allow rt tasks to run in us. |
| * default: 0.95s |
| */ |
| int sysctl_sched_rt_runtime = 950000; |
| |
| static inline u64 global_rt_period(void) |
| { |
| return (u64)sysctl_sched_rt_period * NSEC_PER_USEC; |
| } |
| |
| static inline u64 global_rt_runtime(void) |
| { |
| if (sysctl_sched_rt_runtime < 0) |
| return RUNTIME_INF; |
| |
| return (u64)sysctl_sched_rt_runtime * NSEC_PER_USEC; |
| } |
| |
| #ifndef prepare_arch_switch |
| # define prepare_arch_switch(next) do { } while (0) |
| #endif |
| #ifndef finish_arch_switch |
| # define finish_arch_switch(prev) do { } while (0) |
| #endif |
| |
| static inline int task_current(struct rq *rq, struct task_struct *p) |
| { |
| return rq->curr == p; |
| } |
| |
| static inline int task_running(struct rq *rq, struct task_struct *p) |
| { |
| #ifdef CONFIG_SMP |
| return p->on_cpu; |
| #else |
| return task_current(rq, p); |
| #endif |
| } |
| |
| #ifndef __ARCH_WANT_UNLOCKED_CTXSW |
| static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next) |
| { |
| #ifdef CONFIG_SMP |
| /* |
| * We can optimise this out completely for !SMP, because the |
| * SMP rebalancing from interrupt is the only thing that cares |
| * here. |
| */ |
| next->on_cpu = 1; |
| #endif |
| } |
| |
| static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev) |
| { |
| #ifdef CONFIG_SMP |
| /* |
| * After ->on_cpu is cleared, the task can be moved to a different CPU. |
| * We must ensure this doesn't happen until the switch is completely |
| * finished. |
| */ |
| smp_wmb(); |
| prev->on_cpu = 0; |
| #endif |
| #ifdef CONFIG_DEBUG_SPINLOCK |
| /* this is a valid case when another task releases the spinlock */ |
| rq->lock.owner = current; |
| #endif |
| /* |
| * If we are tracking spinlock dependencies then we have to |
| * fix up the runqueue lock - which gets 'carried over' from |
| * prev into current: |
| */ |
| spin_acquire(&rq->lock.dep_map, 0, 0, _THIS_IP_); |
| |
| raw_spin_unlock_irq(&rq->lock); |
| } |
| |
| #else /* __ARCH_WANT_UNLOCKED_CTXSW */ |
| static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next) |
| { |
| #ifdef CONFIG_SMP |
| /* |
| * We can optimise this out completely for !SMP, because the |
| * SMP rebalancing from interrupt is the only thing that cares |
| * here. |
| */ |
| next->on_cpu = 1; |
| #endif |
| #ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW |
| raw_spin_unlock_irq(&rq->lock); |
| #else |
| raw_spin_unlock(&rq->lock); |
| #endif |
| } |
| |
| static inline void finish_lock_switch(struct rq *rq, struct task_struct *prev) |
| { |
| #ifdef CONFIG_SMP |
| /* |
| * After ->on_cpu is cleared, the task can be moved to a different CPU. |
| * We must ensure this doesn't happen until the switch is completely |
| * finished. |
| */ |
| smp_wmb(); |
| prev->on_cpu = 0; |
| #endif |
| #ifndef __ARCH_WANT_INTERRUPTS_ON_CTXSW |
| local_irq_enable(); |
| #endif |
| } |
| #endif /* __ARCH_WANT_UNLOCKED_CTXSW */ |
| |
| /* |
| * __task_rq_lock - lock the rq @p resides on. |
| */ |
| static inline struct rq *__task_rq_lock(struct task_struct *p) |
| __acquires(rq->lock) |
| { |
| struct rq *rq; |
| |
| lockdep_assert_held(&p->pi_lock); |
| |
| for (;;) { |
| rq = task_rq(p); |
| raw_spin_lock(&rq->lock); |
| if (likely(rq == task_rq(p))) |
| return rq; |
| raw_spin_unlock(&rq->lock); |
| } |
| } |
| |
| /* |
| * task_rq_lock - lock p->pi_lock and lock the rq @p resides on. |
| */ |
| static struct rq *task_rq_lock(struct task_struct *p, unsigned long *flags) |
| __acquires(p->pi_lock) |
| __acquires(rq->lock) |
| { |
| struct rq *rq; |
| |
| for (;;) { |
| raw_spin_lock_irqsave(&p->pi_lock, *flags); |
| rq = task_rq(p); |
| raw_spin_lock(&rq->lock); |
| if (likely(rq == task_rq(p))) |
| return rq; |
| raw_spin_unlock(&rq->lock); |
| raw_spin_unlock_irqrestore(&p->pi_lock, *flags); |
| } |
| } |
| |
| static void __task_rq_unlock(struct rq *rq) |
| __releases(rq->lock) |
| { |
| raw_spin_unlock(&rq->lock); |
| } |
| |
| static inline void |
| task_rq_unlock(struct rq *rq, struct task_struct *p, unsigned long *flags) |
| __releases(rq->lock) |
| __releases(p->pi_lock) |
| { |
| raw_spin_unlock(&rq->lock); |
| raw_spin_unlock_irqrestore(&p->pi_lock, *flags); |
| } |
| |
| /* |
| * this_rq_lock - lock this runqueue and disable interrupts. |
| */ |
| static struct rq *this_rq_lock(void) |
| __acquires(rq->lock) |
| { |
| struct rq *rq; |
| |
| local_irq_disable(); |
| rq = this_rq(); |
| raw_spin_lock(&rq->lock); |
| |
| return rq; |
| } |
| |
| #ifdef CONFIG_SCHED_HRTICK |
| /* |
| * Use HR-timers to deliver accurate preemption points. |
| * |
| * Its all a bit involved since we cannot program an hrt while holding the |
| * rq->lock. So what we do is store a state in in rq->hrtick_* and ask for a |
| * reschedule event. |
| * |
| * When we get rescheduled we reprogram the hrtick_timer outside of the |
| * rq->lock. |
| */ |
| |
| /* |
| * Use hrtick when: |
| * - enabled by features |
| * - hrtimer is actually high res |
| */ |
| static inline int hrtick_enabled(struct rq *rq) |
| { |
| if (!sched_feat(HRTICK)) |
| return 0; |
| if (!cpu_active(cpu_of(rq))) |
| return 0; |
| return hrtimer_is_hres_active(&rq->hrtick_timer); |
| } |
| |
| static void hrtick_clear(struct rq *rq) |
| { |
| if (hrtimer_active(&rq->hrtick_timer)) |
| hrtimer_cancel(&rq->hrtick_timer); |
| } |
| |
| /* |
| * High-resolution timer tick. |
| * Runs from hardirq context with interrupts disabled. |
| */ |
| static enum hrtimer_restart hrtick(struct hrtimer *timer) |
| { |
| struct rq *rq = container_of(timer, struct rq, hrtick_timer); |
| |
| WARN_ON_ONCE(cpu_of(rq) != smp_processor_id()); |
| |
| raw_spin_lock(&rq->lock); |
| update_rq_clock(rq); |
| rq->curr->sched_class->task_tick(rq, rq->curr, 1); |
| raw_spin_unlock(&rq->lock); |
| |
| return HRTIMER_NORESTART; |
| } |
| |
| #ifdef CONFIG_SMP |
| /* |
| * called from hardirq (IPI) context |
| */ |
| static void __hrtick_start(void *arg) |
| { |
| struct rq *rq = arg; |
| |
| raw_spin_lock(&rq->lock); |
| hrtimer_restart(&rq->hrtick_timer); |
| rq->hrtick_csd_pending = 0; |
| raw_spin_unlock(&rq->lock); |
| } |
| |
| /* |
| * Called to set the hrtick timer state. |
| * |
| * called with rq->lock held and irqs disabled |
| */ |
| static void hrtick_start(struct rq *rq, u64 delay) |
| { |
| struct hrtimer *timer = &rq->hrtick_timer; |
| ktime_t time = ktime_add_ns(timer->base->get_time(), delay); |
| |
| hrtimer_set_expires(timer, time); |
| |
| if (rq == this_rq()) { |
| hrtimer_restart(timer); |
| } else if (!rq->hrtick_csd_pending) { |
| __smp_call_function_single(cpu_of(rq), &rq->hrtick_csd, 0); |
| rq->hrtick_csd_pending = 1; |
| } |
| } |
| |
| static int |
| hotplug_hrtick(struct notifier_block *nfb, unsigned long action, void *hcpu) |
| { |
| int cpu = (int)(long)hcpu; |
| |
| switch (action) { |
| case CPU_UP_CANCELED: |
| case CPU_UP_CANCELED_FROZEN: |
| case CPU_DOWN_PREPARE: |
| case CPU_DOWN_PREPARE_FROZEN: |
| case CPU_DEAD: |
| case CPU_DEAD_FROZEN: |
| hrtick_clear(cpu_rq(cpu)); |
| return NOTIFY_OK; |
| } |
| |
| return NOTIFY_DONE; |
| } |
| |
| static __init void init_hrtick(void) |
| { |
| hotcpu_notifier(hotplug_hrtick, 0); |
| } |
| #else |
| /* |
| * Called to set the hrtick timer state. |
| * |
| * called with rq->lock held and irqs disabled |
| */ |
| static void hrtick_start(struct rq *rq, u64 delay) |
| { |
| __hrtimer_start_range_ns(&rq->hrtick_timer, ns_to_ktime(delay), 0, |
| HRTIMER_MODE_REL_PINNED, 0); |
| } |
| |
| static inline void init_hrtick(void) |
| { |
| } |
| #endif /* CONFIG_SMP */ |
| |
| static void init_rq_hrtick(struct rq *rq) |
| { |
| #ifdef CONFIG_SMP |
| rq->hrtick_csd_pending = 0; |
| |
| rq->hrtick_csd.flags = 0; |
| rq->hrtick_csd.func = __hrtick_start; |
| rq->hrtick_csd.info = rq; |
| #endif |
| |
| hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); |
| rq->hrtick_timer.function = hrtick; |
| } |
| #else /* CONFIG_SCHED_HRTICK */ |
| static inline void hrtick_clear(struct rq *rq) |
| { |
| } |
| |
| static inline void init_rq_hrtick(struct rq *rq) |
| { |
| } |
| |
| static inline void init_hrtick(void) |
| { |
| } |
| #endif /* CONFIG_SCHED_HRTICK */ |
| |
| /* |
| * resched_task - mark a task 'to be rescheduled now'. |
| * |
| * On UP this means the setting of the need_resched flag, on SMP it |
| * might also involve a cross-CPU call to trigger the scheduler on |
| * the target CPU. |
| */ |
| #ifdef CONFIG_SMP |
| |
| #ifndef tsk_is_polling |
| #define tsk_is_polling(t) test_tsk_thread_flag(t, TIF_POLLING_NRFLAG) |
| #endif |
| |
| static void resched_task(struct task_struct *p) |
| { |
| int cpu; |
| |
| assert_raw_spin_locked(&task_rq(p)->lock); |
| |
| if (test_tsk_need_resched(p)) |
| return; |
| |
| set_tsk_need_resched(p); |
| |
| cpu = task_cpu(p); |
| if (cpu == smp_processor_id()) |
| return; |
| |
| /* NEED_RESCHED must be visible before we test polling */ |
| smp_mb(); |
| if (!tsk_is_polling(p)) |
| smp_send_reschedule(cpu); |
| } |
| |
| static void resched_cpu(int cpu) |
| { |
| struct rq *rq = cpu_rq(cpu); |
| unsigned long flags; |
| |
| if (!raw_spin_trylock_irqsave(&rq->lock, flags)) |
| return; |
| resched_task(cpu_curr(cpu)); |
| raw_spin_unlock_irqrestore(&rq->lock, flags); |
| } |
| |
| #ifdef CONFIG_NO_HZ |
| /* |
| * In the semi idle case, use the nearest busy cpu for migrating timers |
| * from an idle cpu. This is good for power-savings. |
| * |
| * We don't do similar optimization for completely idle system, as |
| * selecting an idle cpu will add more delays to the timers than intended |
| * (as that cpu's timer base may not be uptodate wrt jiffies etc). |
| */ |
| int get_nohz_timer_target(void) |
| { |
| int cpu = smp_processor_id(); |
| int i; |
| struct sched_domain *sd; |
| |
| rcu_read_lock(); |
| for_each_domain(cpu, sd) { |
| for_each_cpu(i, sched_domain_span(sd)) { |
| if (!idle_cpu(i)) { |
| cpu = i; |
| goto unlock; |
| } |
| } |
| } |
| unlock: |
| rcu_read_unlock(); |
| return cpu; |
| } |
| /* |
| * When add_timer_on() enqueues a timer into the timer wheel of an |
| * idle CPU then this timer might expire before the next timer event |
| * which is scheduled to wake up that CPU. In case of a completely |
| * idle system the next event might even be infinite time into the |
| * future. wake_up_idle_cpu() ensures that the CPU is woken up and |
| * leaves the inner idle loop so the newly added timer is taken into |
| * account when the CPU goes back to idle and evaluates the timer |
| * wheel for the next timer event. |
| */ |
| void wake_up_idle_cpu(int cpu) |
| { |
| struct rq *rq = cpu_rq(cpu); |
| |
| if (cpu == smp_processor_id()) |
| return; |
| |
| /* |
| * This is safe, as this function is called with the timer |
| * wheel base lock of (cpu) held. When the CPU is on the way |
| * to idle and has not yet set rq->curr to idle then it will |
| * be serialized on the timer wheel base lock and take the new |
| * timer into account automatically. |
| */ |
| if (rq->curr != rq->idle) |
| return; |
| |
| /* |
| * We can set TIF_RESCHED on the idle task of the other CPU |
| * lockless. The worst case is that the other CPU runs the |
| * idle task through an additional NOOP schedule() |
| */ |
| set_tsk_need_resched(rq->idle); |
| |
| /* NEED_RESCHED must be visible before we test polling */ |
| smp_mb(); |
| if (!tsk_is_polling(rq->idle)) |
| smp_send_reschedule(cpu); |
| } |
| |
| static inline bool got_nohz_idle_kick(void) |
| { |
| return idle_cpu(smp_processor_id()) && this_rq()->nohz_balance_kick; |
| } |
| |
| #else /* CONFIG_NO_HZ */ |
| |
| static inline bool got_nohz_idle_kick(void) |
| { |
| return false; |
| } |
| |
| #endif /* CONFIG_NO_HZ */ |
| |
| static u64 sched_avg_period(void) |
| { |
| return (u64)sysctl_sched_time_avg * NSEC_PER_MSEC / 2; |
| } |
| |
| static void sched_avg_update(struct rq *rq) |
| { |
| s64 period = sched_avg_period(); |
| |
| while ((s64)(rq->clock - rq->age_stamp) > period) { |
| /* |
| * Inline assembly required to prevent the compiler |
| * optimising this loop into a divmod call. |
| * See __iter_div_u64_rem() for another example of this. |
| */ |
| asm("" : "+rm" (rq->age_stamp)); |
| rq->age_stamp += period; |
| rq->rt_avg /= 2; |
| } |
| } |
| |
| static void sched_rt_avg_update(struct rq *rq, u64 rt_delta) |
| { |
| rq->rt_avg += rt_delta; |
| sched_avg_update(rq); |
| } |
| |
| #else /* !CONFIG_SMP */ |
| static void resched_task(struct task_struct *p) |
| { |
| assert_raw_spin_locked(&task_rq(p)->lock); |
| set_tsk_need_resched(p); |
| } |
| |
| static void sched_rt_avg_update(struct rq *rq, u64 rt_delta) |
| { |
| } |
| |
| static void sched_avg_update(struct rq *rq) |
| { |
| } |
| #endif /* CONFIG_SMP */ |
| |
| #if BITS_PER_LONG == 32 |
| # define WMULT_CONST (~0UL) |
| #else |
| # define WMULT_CONST (1UL << 32) |
| #endif |
| |
| #define WMULT_SHIFT 32 |
| |
| /* |
| * Shift right and round: |
| */ |
| #define SRR(x, y) (((x) + (1UL << ((y) - 1))) >> (y)) |
| |
| /* |
| * delta *= weight / lw |
| */ |
| static unsigned long |
| calc_delta_mine(unsigned long delta_exec, unsigned long weight, |
| struct load_weight *lw) |
| { |
| u64 tmp; |
| |
| /* |
| * weight can be less than 2^SCHED_LOAD_RESOLUTION for task group sched |
| * entities since MIN_SHARES = 2. Treat weight as 1 if less than |
| * 2^SCHED_LOAD_RESOLUTION. |
| */ |
| if (likely(weight > (1UL << SCHED_LOAD_RESOLUTION))) |
| tmp = (u64)delta_exec * scale_load_down(weight); |
| else |
| tmp = (u64)delta_exec; |
| |
| if (!lw->inv_weight) { |
| unsigned long w = scale_load_down(lw->weight); |
| |
| if (BITS_PER_LONG > 32 && unlikely(w >= WMULT_CONST)) |
| lw->inv_weight = 1; |
| else if (unlikely(!w)) |
| lw->inv_weight = WMULT_CONST; |
| else |
| lw->inv_weight = WMULT_CONST / w; |
| } |
| |
| /* |
| * Check whether we'd overflow the 64-bit multiplication: |
| */ |
| if (unlikely(tmp > WMULT_CONST)) |
| tmp = SRR(SRR(tmp, WMULT_SHIFT/2) * lw->inv_weight, |
| WMULT_SHIFT/2); |
| else |
| tmp = SRR(tmp * lw->inv_weight, WMULT_SHIFT); |
| |
| return (unsigned long)min(tmp, (u64)(unsigned long)LONG_MAX); |
| } |
| |
| static inline void update_load_add(struct load_weight *lw, unsigned long inc) |
| { |
| lw->weight += inc; |
| lw->inv_weight = 0; |
| } |
| |
| static inline void update_load_sub(struct load_weight *lw, unsigned long dec) |
| { |
| lw->weight -= dec; |
| lw->inv_weight = 0; |
| } |
| |
| static inline void update_load_set(struct load_weight *lw, unsigned long w) |
| { |
| lw->weight = w; |
| lw->inv_weight = 0; |
| } |
| |
| /* |
| * To aid in avoiding the subversion of "niceness" due to uneven distribution |
| * of tasks with abnormal "nice" values across CPUs the contribution that |
| * each task makes to its run queue's load is weighted according to its |
| * scheduling class and "nice" value. For SCHED_NORMAL tasks this is just a |
| * scaled version of the new time slice allocation that they receive on time |
| * slice expiry etc. |
| */ |
| |
| #define WEIGHT_IDLEPRIO 3 |
| #define WMULT_IDLEPRIO 1431655765 |
| |
| /* |
| * Nice levels are multiplicative, with a gentle 10% change for every |
| * nice level changed. I.e. when a CPU-bound task goes from nice 0 to |
| * nice 1, it will get ~10% less CPU time than another CPU-bound task |
| * that remained on nice 0. |
| * |
| * The "10% effect" is relative and cumulative: from _any_ nice level, |
| * if you go up 1 level, it's -10% CPU usage, if you go down 1 level |
| * it's +10% CPU usage. (to achieve that we use a multiplier of 1.25. |
| * If a task goes up by ~10% and another task goes down by ~10% then |
| * the relative distance between them is ~25%.) |
| */ |
| static const int prio_to_weight[40] = { |
| /* -20 */ 88761, 71755, 56483, 46273, 36291, |
| /* -15 */ 29154, 23254, 18705, 14949, 11916, |
| /* -10 */ 9548, 7620, 6100, 4904, 3906, |
| /* -5 */ 3121, 2501, 1991, 1586, 1277, |
| /* 0 */ 1024, 820, 655, 526, 423, |
| /* 5 */ 335, 272, 215, 172, 137, |
| /* 10 */ 110, 87, 70, 56, 45, |
| /* 15 */ 36, 29, 23, 18, 15, |
| }; |
| |
| /* |
| * Inverse (2^32/x) values of the prio_to_weight[] array, precalculated. |
| * |
| * In cases where the weight does not change often, we can use the |
| * precalculated inverse to speed up arithmetics by turning divisions |
| * into multiplications: |
| */ |
| static const u32 prio_to_wmult[40] = { |
| /* -20 */ 48388, 59856, 76040, 92818, 118348, |
| /* -15 */ 147320, 184698, 229616, 287308, 360437, |
| /* -10 */ 449829, 563644, 704093, 875809, 1099582, |
| /* -5 */ 1376151, 1717300, 2157191, 2708050, 3363326, |
| /* 0 */ 4194304, 5237765, 6557202, 8165337, 10153587, |
| /* 5 */ 12820798, 15790321, 19976592, 24970740, 31350126, |
| /* 10 */ 39045157, 49367440, 61356676, 76695844, 95443717, |
| /* 15 */ 119304647, 148102320, 186737708, 238609294, 286331153, |
| }; |
| |
| /* Time spent by the tasks of the cpu accounting group executing in ... */ |
| enum cpuacct_stat_index { |
| CPUACCT_STAT_USER, /* ... user mode */ |
| CPUACCT_STAT_SYSTEM, /* ... kernel mode */ |
| |
| CPUACCT_STAT_NSTATS, |
| }; |
| |
| #ifdef CONFIG_CGROUP_CPUACCT |
| static void cpuacct_charge(struct task_struct *tsk, u64 cputime); |
| static void cpuacct_update_stats(struct task_struct *tsk, |
| enum cpuacct_stat_index idx, cputime_t val); |
| #else |
| static inline void cpuacct_charge(struct task_struct *tsk, u64 cputime) {} |
| static inline void cpuacct_update_stats(struct task_struct *tsk, |
| enum cpuacct_stat_index idx, cputime_t val) {} |
| #endif |
| |
| static inline void inc_cpu_load(struct rq *rq, unsigned long load) |
| { |
| update_load_add(&rq->load, load); |
| } |
| |
| static inline void dec_cpu_load(struct rq *rq, unsigned long load) |
| { |
| update_load_sub(&rq->load, load); |
| } |
| |
| #if defined(CONFIG_RT_GROUP_SCHED) || (defined(CONFIG_FAIR_GROUP_SCHED) && \ |
| (defined(CONFIG_SMP) || defined(CONFIG_CFS_BANDWIDTH))) |
| typedef int (*tg_visitor)(struct task_group *, void *); |
| |
| /* |
| * Iterate task_group tree rooted at *from, calling @down when first entering a |
| * node and @up when leaving it for the final time. |
| * |
| * Caller must hold rcu_lock or sufficient equivalent. |
| */ |
| static int walk_tg_tree_from(struct task_group *from, |
| tg_visitor down, tg_visitor up, void *data) |
| { |
| struct task_group *parent, *child; |
| int ret; |
| |
| parent = from; |
| |
| down: |
| ret = (*down)(parent, data); |
| if (ret) |
| goto out; |
| list_for_each_entry_rcu(child, &parent->children, siblings) { |
| parent = child; |
| goto down; |
| |
| up: |
| continue; |
| } |
| ret = (*up)(parent, data); |
| if (ret || parent == from) |
| goto out; |
| |
| child = parent; |
| parent = parent->parent; |
| if (parent) |
| goto up; |
| out: |
| return ret; |
| } |
| |
| /* |
| * Iterate the full tree, calling @down when first entering a node and @up when |
| * leaving it for the final time. |
| * |
| * Caller must hold rcu_lock or sufficient equivalent. |
| */ |
| |
| static inline int walk_tg_tree(tg_visitor down, tg_visitor up, void *data) |
| { |
| return walk_tg_tree_from(&root_task_group, down, up, data); |
| } |
| |
| static int tg_nop(struct task_group *tg, void *data) |
| { |
| return 0; |
| } |
| #endif |
| |
| #ifdef CONFIG_SMP |
| /* Used instead of source_load when we know the type == 0 */ |
| static unsigned long weighted_cpuload(const int cpu) |
| { |
| return cpu_rq(cpu)->load.weight; |
| } |
| |
| /* |
| * Return a low guess at the load of a migration-source cpu weighted |
| * according to the scheduling class and "nice" value. |
| * |
| * We want to under-estimate the load of migration sources, to |
| * balance conservatively. |
| */ |
| static unsigned long source_load(int cpu, int type) |
| { |
| struct rq *rq = cpu_rq(cpu); |
| unsigned long total = weighted_cpuload(cpu); |
| |
| if (type == 0 || !sched_feat(LB_BIAS)) |
| return total; |
| |
| return min(rq->cpu_load[type-1], total); |
| } |
| |
| /* |
| * Return a high guess at the load of a migration-target cpu weighted |
| * according to the scheduling class and "nice" value. |
| */ |
| static unsigned long target_load(int cpu, int type) |
| { |
| struct rq *rq = cpu_rq(cpu); |
| unsigned long total = weighted_cpuload(cpu); |
| |
| if (type == 0 || !sched_feat(LB_BIAS)) |
| return total; |
| |
| return max(rq->cpu_load[type-1], total); |
| } |
| |
| static unsigned long power_of(int cpu) |
| { |
| return cpu_rq(cpu)->cpu_power; |
| } |
| |
| static int task_hot(struct task_struct *p, u64 now, struct sched_domain *sd); |
| |
| static unsigned long cpu_avg_load_per_task(int cpu) |
| { |
| struct rq *rq = cpu_rq(cpu); |
| unsigned long nr_running = ACCESS_ONCE(rq->nr_running); |
| |
| if (nr_running) |
| return rq->load.weight / nr_running; |
| |
| return 0; |
| } |
| |
| #ifdef CONFIG_PREEMPT |
| |
| static void double_rq_lock(struct rq *rq1, struct rq *rq2); |
| |
| /* |
| * fair double_lock_balance: Safely acquires both rq->locks in a fair |
| * way at the expense of forcing extra atomic operations in all |
| * invocations. This assures that the double_lock is acquired using the |
| * same underlying policy as the spinlock_t on this architecture, which |
| * reduces latency compared to the unfair variant below. However, it |
| * also adds more overhead and therefore may reduce throughput. |
| */ |
| static inline int _double_lock_balance(struct rq *this_rq, struct rq *busiest) |
| __releases(this_rq->lock) |
| __acquires(busiest->lock) |
| __acquires(this_rq->lock) |
| { |
| raw_spin_unlock(&this_rq->lock); |
| double_rq_lock(this_rq, busiest); |
| |
| return 1; |
| } |
| |
| #else |
| /* |
| * Unfair double_lock_balance: Optimizes throughput at the expense of |
| * latency by eliminating extra atomic operations when the locks are |
| * already in proper order on entry. This favors lower cpu-ids and will |
| * grant the double lock to lower cpus over higher ids under contention, |
| * regardless of entry order into the function. |
| */ |
| static int _double_lock_balance(struct rq *this_rq, struct rq *busiest) |
| __releases(this_rq->lock) |
| __acquires(busiest->lock) |
| __acquires(this_rq->lock) |
| { |
| int ret = 0; |
| |
| if (unlikely(!raw_spin_trylock(&busiest->lock))) { |
| if (busiest < this_rq) { |
| raw_spin_unlock(&this_rq->lock); |
| raw_spin_lock(&busiest->lock); |
| raw_spin_lock_nested(&this_rq->lock, |
| SINGLE_DEPTH_NESTING); |
| ret = 1; |
| } else |
| raw_spin_lock_nested(&busiest->lock, |
| SINGLE_DEPTH_NESTING); |
| } |
| return ret; |
| } |
| |
| #endif /* CONFIG_PREEMPT */ |
| |
| /* |
| * double_lock_balance - lock the busiest runqueue, this_rq is locked already. |
| */ |
| static int double_lock_balance(struct rq *this_rq, struct rq *busiest) |
| { |
| if (unlikely(!irqs_disabled())) { |
| /* printk() doesn't work good under rq->lock */ |
| raw_spin_unlock(&this_rq->lock); |
| BUG_ON(1); |
| } |
| |
| return _double_lock_balance(this_rq, busiest); |
| } |
| |
| static inline void double_unlock_balance(struct rq *this_rq, struct rq *busiest) |
| __releases(busiest->lock) |
| { |
| raw_spin_unlock(&busiest->lock); |
| lock_set_subclass(&this_rq->lock.dep_map, 0, _RET_IP_); |
| } |
| |
| /* |
| * double_rq_lock - safely lock two runqueues |
| * |
| * Note this does not disable interrupts like task_rq_lock, |
| * you need to do so manually before calling. |
| */ |
| static void double_rq_lock(struct rq *rq1, struct rq *rq2) |
| __acquires(rq1->lock) |
| __acquires(rq2->lock) |
| { |
| BUG_ON(!irqs_disabled()); |
| if (rq1 == rq2) { |
| raw_spin_lock(&rq1->lock); |
| __acquire(rq2->lock); /* Fake it out ;) */ |
| } else { |
| if (rq1 < rq2) { |
| raw_spin_lock(&rq1->lock); |
| raw_spin_lock_nested(&rq2->lock, SINGLE_DEPTH_NESTING); |
| } else { |
| raw_spin_lock(&rq2->lock); |
| raw_spin_lock_nested(&rq1->lock, SINGLE_DEPTH_NESTING); |
| } |
| } |
| } |
| |
| /* |
| * double_rq_unlock - safely unlock two runqueues |
| * |
| * Note this does not restore interrupts like task_rq_unlock, |
| * you need to do so manually after calling. |
| */ |
| static void double_rq_unlock(struct rq *rq1, struct rq *rq2) |
| __releases(rq1->lock) |
| __releases(rq2->lock) |
| { |
| raw_spin_unlock(&rq1->lock); |
| if (rq1 != rq2) |
| raw_spin_unlock(&rq2->lock); |
| else |
| __release(rq2->lock); |
| } |
| |
| #else /* CONFIG_SMP */ |
| |
| /* |
| * double_rq_lock - safely lock two runqueues |
| * |
| * Note this does not disable interrupts like task_rq_lock, |
| * you need to do so manually before calling. |
| */ |
| static void double_rq_lock(struct rq *rq1, struct rq *rq2) |
| __acquires(rq1->lock) |
| __acquires(rq2->lock) |
| { |
| BUG_ON(!irqs_disabled()); |
| BUG_ON(rq1 != rq2); |
| raw_spin_lock(&rq1->lock); |
| __acquire(rq2->lock); /* Fake it out ;) */ |
| } |
| |
| /* |
| * double_rq_unlock - safely unlock two runqueues |
| * |
| * Note this does not restore interrupts like task_rq_unlock, |
| * you need to do so manually after calling. |
| */ |
| static void double_rq_unlock(struct rq *rq1, struct rq *rq2) |
| __releases(rq1->lock) |
| __releases(rq2->lock) |
| { |
| BUG_ON(rq1 != rq2); |
| raw_spin_unlock(&rq1->lock); |
| __release(rq2->lock); |
| } |
| |
| #endif |
| |
| static void update_sysctl(void); |
| static int get_update_sysctl_factor(void); |
| static void update_idle_cpu_load(struct rq *this_rq); |
| |
| static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu) |
| { |
| set_task_rq(p, cpu); |
| #ifdef CONFIG_SMP |
| /* |
| * After ->cpu is set up to a new value, task_rq_lock(p, ...) can be |
| * successfully executed on another CPU. We must ensure that updates of |
| * per-task data have been completed by this moment. |
| */ |
| smp_wmb(); |
| task_thread_info(p)->cpu = cpu; |
| #endif |
| } |
| |
| static const struct sched_class rt_sched_class; |
| |
| #define sched_class_highest (&stop_sched_class) |
| #define for_each_class(class) \ |
| for (class = sched_class_highest; class; class = class->next) |
| |
| #include "sched_stats.h" |
| |
| static void inc_nr_running(struct rq *rq) |
| { |
| rq->nr_running++; |
| } |
| |
| static void dec_nr_running(struct rq *rq) |
| { |
| rq->nr_running--; |
| } |
| |
| static void set_load_weight(struct task_struct *p) |
| { |
| int prio = p->static_prio - MAX_RT_PRIO; |
| struct load_weight *load = &p->se.load; |
| |
| /* |
| * SCHED_IDLE tasks get minimal weight: |
| */ |
| if (p->policy == SCHED_IDLE) { |
| load->weight = scale_load(WEIGHT_IDLEPRIO); |
| load->inv_weight = WMULT_IDLEPRIO; |
| return; |
| } |
| |
| load->weight = scale_load(prio_to_weight[prio]); |
| load->inv_weight = prio_to_wmult[prio]; |
| } |
| |
| static void enqueue_task(struct rq *rq, struct task_struct *p, int flags) |
| { |
| update_rq_clock(rq); |
| sched_info_queued(p); |
| p->sched_class->enqueue_task(rq, p, flags); |
| } |
| |
| static void dequeue_task(struct rq *rq, struct task_struct *p, int flags) |
| { |
| update_rq_clock(rq); |
| sched_info_dequeued(p); |
| p->sched_class->dequeue_task(rq, p, flags); |
| } |
| |
| /* |
| * activate_task - move a task to the runqueue. |
| */ |
| static void activate_task(struct rq *rq, struct task_struct *p, int flags) |
| { |
| if (task_contributes_to_load(p)) |
| rq->nr_uninterruptible--; |
| |
| enqueue_task(rq, p, flags); |
| } |
| |
| /* |
| * deactivate_task - remove a task from the runqueue. |
| */ |
| static void deactivate_task(struct rq *rq, struct task_struct *p, int flags) |
| { |
| if (task_contributes_to_load(p)) |
| rq->nr_uninterruptible++; |
| |
| dequeue_task(rq, p, flags); |
| } |
| |
| #ifdef CONFIG_IRQ_TIME_ACCOUNTING |
| |
| /* |
| * There are no locks covering percpu hardirq/softirq time. |
| * They are only modified in account_system_vtime, on corresponding CPU |
| * with interrupts disabled. So, writes are safe. |
| * They are read and saved off onto struct rq in update_rq_clock(). |
| * This may result in other CPU reading this CPU's irq time and can |
| * race with irq/account_system_vtime on this CPU. We would either get old |
| * or new value with a side effect of accounting a slice of irq time to wrong |
| * task when irq is in progress while we read rq->clock. That is a worthy |
| * compromise in place of having locks on each irq in account_system_time. |
| */ |
| static DEFINE_PER_CPU(u64, cpu_hardirq_time); |
| static DEFINE_PER_CPU(u64, cpu_softirq_time); |
| |
| static DEFINE_PER_CPU(u64, irq_start_time); |
| static int sched_clock_irqtime; |
| |
| void enable_sched_clock_irqtime(void) |
| { |
| sched_clock_irqtime = 1; |
| } |
| |
| void disable_sched_clock_irqtime(void) |
| { |
| sched_clock_irqtime = 0; |
| } |
| |
| #ifndef CONFIG_64BIT |
| static DEFINE_PER_CPU(seqcount_t, irq_time_seq); |
| |
| static inline void irq_time_write_begin(void) |
| { |
| __this_cpu_inc(irq_time_seq.sequence); |
| smp_wmb(); |
| } |
| |
| static inline void irq_time_write_end(void) |
| { |
| smp_wmb(); |
| __this_cpu_inc(irq_time_seq.sequence); |
| } |
| |
| static inline u64 irq_time_read(int cpu) |
| { |
| u64 irq_time; |
| unsigned seq; |
| |
| do { |
| seq = read_seqcount_begin(&per_cpu(irq_time_seq, cpu)); |
| irq_time = per_cpu(cpu_softirq_time, cpu) + |
| per_cpu(cpu_hardirq_time, cpu); |
| } while (read_seqcount_retry(&per_cpu(irq_time_seq, cpu), seq)); |
| |
| return irq_time; |
| } |
| #else /* CONFIG_64BIT */ |
| static inline void irq_time_write_begin(void) |
| { |
| } |
| |
| static inline void irq_time_write_end(void) |
| { |
| } |
| |
| static inline u64 irq_time_read(int cpu) |
| { |
| return per_cpu(cpu_softirq_time, cpu) + per_cpu(cpu_hardirq_time, cpu); |
| } |
| #endif /* CONFIG_64BIT */ |
| |
| /* |
| * Called before incrementing preempt_count on {soft,}irq_enter |
| * and before decrementing preempt_count on {soft,}irq_exit. |
| */ |
| void account_system_vtime(struct task_struct *curr) |
| { |
| unsigned long flags; |
| s64 delta; |
| int cpu; |
| |
| if (!sched_clock_irqtime) |
| return; |
| |
| local_irq_save(flags); |
| |
| cpu = smp_processor_id(); |
| delta = sched_clock_cpu(cpu) - __this_cpu_read(irq_start_time); |
| __this_cpu_add(irq_start_time, delta); |
| |
| irq_time_write_begin(); |
| /* |
| * We do not account for softirq time from ksoftirqd here. |
| * We want to continue accounting softirq time to ksoftirqd thread |
| * in that case, so as not to confuse scheduler with a special task |
| * that do not consume any time, but still wants to run. |
| */ |
| if (hardirq_count()) |
| __this_cpu_add(cpu_hardirq_time, delta); |
| else if (in_serving_softirq() && curr != this_cpu_ksoftirqd()) |
| __this_cpu_add(cpu_softirq_time, delta); |
| |
| irq_time_write_end(); |
| local_irq_restore(flags); |
| } |
| EXPORT_SYMBOL_GPL(account_system_vtime); |
| |
| #endif /* CONFIG_IRQ_TIME_ACCOUNTING */ |
| |
| #ifdef CONFIG_PARAVIRT |
| static inline u64 steal_ticks(u64 steal) |
| { |
| if (unlikely(steal > NSEC_PER_SEC)) |
| return div_u64(steal, TICK_NSEC); |
| |
| return __iter_div_u64_rem(steal, TICK_NSEC, &steal); |
| } |
| #endif |
| |
| static void update_rq_clock_task(struct rq *rq, s64 delta) |
| { |
| /* |
| * In theory, the compile should just see 0 here, and optimize out the call |
| * to sched_rt_avg_update. But I don't trust it... |
| */ |
| #if defined(CONFIG_IRQ_TIME_ACCOUNTING) || defined(CONFIG_PARAVIRT_TIME_ACCOUNTING) |
| s64 steal = 0, irq_delta = 0; |
| #endif |
| #ifdef CONFIG_IRQ_TIME_ACCOUNTING |
| irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time; |
| |
| /* |
| * Since irq_time is only updated on {soft,}irq_exit, we might run into |
| * this case when a previous update_rq_clock() happened inside a |
| * {soft,}irq region. |
| * |
| * When this happens, we stop ->clock_task and only update the |
| * prev_irq_time stamp to account for the part that fit, so that a next |
| * update will consume the rest. This ensures ->clock_task is |
| * monotonic. |
| * |
| * It does however cause some slight miss-attribution of {soft,}irq |
| * time, a more accurate solution would be to update the irq_time using |
| * the current rq->clock timestamp, except that would require using |
| * atomic ops. |
| */ |
| if (irq_delta > delta) |
| irq_delta = delta; |
| |
| rq->prev_irq_time += irq_delta; |
| delta -= irq_delta; |
| #endif |
| #ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING |
| if (static_branch((¶virt_steal_rq_enabled))) { |
| u64 st; |
| |
| steal = paravirt_steal_clock(cpu_of(rq)); |
| steal -= rq->prev_steal_time_rq; |
| |
| if (unlikely(steal > delta)) |
| steal = delta; |
| |
| st = steal_ticks(steal); |
| steal = st * TICK_NSEC; |
| |
| rq->prev_steal_time_rq += steal; |
| |
| delta -= steal; |
| } |
| #endif |
| |
| rq->clock_task += delta; |
| |
| #if defined(CONFIG_IRQ_TIME_ACCOUNTING) || defined(CONFIG_PARAVIRT_TIME_ACCOUNTING) |
| if ((irq_delta + steal) && sched_feat(NONTASK_POWER)) |
| sched_rt_avg_update(rq, irq_delta + steal); |
| #endif |
| } |
| |
| #ifdef CONFIG_IRQ_TIME_ACCOUNTING |
| static int irqtime_account_hi_update(void) |
| { |
| struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat; |
| unsigned long flags; |
| u64 latest_ns; |
| int ret = 0; |
| |
| local_irq_save(flags); |
| latest_ns = this_cpu_read(cpu_hardirq_time); |
| if (cputime64_gt(nsecs_to_cputime64(latest_ns), cpustat->irq)) |
| ret = 1; |
| local_irq_restore(flags); |
| return ret; |
| } |
| |
| static int irqtime_account_si_update(void) |
| { |
| struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat; |
| unsigned long flags; |
| u64 latest_ns; |
| int ret = 0; |
| |
| local_irq_save(flags); |
| latest_ns = this_cpu_read(cpu_softirq_time); |
| if (cputime64_gt(nsecs_to_cputime64(latest_ns), cpustat->softirq)) |
| ret = 1; |
| local_irq_restore(flags); |
| return ret; |
| } |
| |
| #else /* CONFIG_IRQ_TIME_ACCOUNTING */ |
| |
| #define sched_clock_irqtime (0) |
| |
| #endif |
| |
| #include "sched_idletask.c" |
| #include "sched_fair.c" |
| #include "sched_rt.c" |
| #include "sched_autogroup.c" |
| #include "sched_stoptask.c" |
| #ifdef CONFIG_SCHED_DEBUG |
| # include "sched_debug.c" |
| #endif |
| |
| void sched_set_stop_task(int cpu, struct task_struct *stop) |
| { |
| struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 }; |
| struct task_struct *old_stop = cpu_rq(cpu)->stop; |
| |
| if (stop) { |
| /* |
| * Make it appear like a SCHED_FIFO task, its something |
| * userspace knows about and won't get confused about. |
| * |
| * Also, it will make PI more or less work without too |
| * much confusion -- but then, stop work should not |
| * rely on PI working anyway. |
| */ |
| sched_setscheduler_nocheck(stop, SCHED_FIFO, ¶m); |
| |
| stop->sched_class = &stop_sched_class; |
| } |
| |
| cpu_rq(cpu)->stop = stop; |
| |
| if (old_stop) { |
| /* |
| * Reset it back to a normal scheduling class so that |
| * it can die in pieces. |
| */ |
| old_stop->sched_class = &rt_sched_class; |
| } |
| } |
| |
| /* |
| * __normal_prio - return the priority that is based on the static prio |
| */ |
| static inline int __normal_prio(struct task_struct *p) |
| { |
| return p->static_prio; |
| } |
| |
| /* |
| * Calculate the expected normal priority: i.e. priority |
| * without taking RT-inheritance into account. Might be |
| * boosted by interactivity modifiers. Changes upon fork, |
| * setprio syscalls, and whenever the interactivity |
| * estimator recalculates. |
| */ |
| static inline int normal_prio(struct task_struct *p) |
| { |
| int prio; |
| |
| if (task_has_rt_policy(p)) |
| prio = MAX_RT_PRIO-1 - p->rt_priority; |
| else |
| prio = __normal_prio(p); |
| return prio; |
| } |
| |
| /* |
| * Calculate the current priority, i.e. the priority |
| * taken into account by the scheduler. This value might |
| * be boosted by RT tasks, or might be boosted by |
| * interactivity modifiers. Will be RT if the task got |
| * RT-boosted. If not then it returns p->normal_prio. |
| */ |
| static int effective_prio(struct task_struct *p) |
| { |
| p->normal_prio = normal_prio(p); |
| /* |
| * If we are RT tasks or we were boosted to RT priority, |
| * keep the priority unchanged. Otherwise, update priority |
| * to the normal priority: |
| */ |
| if (!rt_prio(p->prio)) |
| return p->normal_prio; |
| return p->prio; |
| } |
| |
| /** |
| * task_curr - is this task currently executing on a CPU? |
| * @p: the task in question. |
| */ |
| inline int task_curr(const struct task_struct *p) |
| { |
| return cpu_curr(task_cpu(p)) == p; |
| } |
| |
| static inline void check_class_changed(struct rq *rq, struct task_struct *p, |
| const struct sched_class *prev_class, |
| int oldprio) |
| { |
| if (prev_class != p->sched_class) { |
| if (prev_class->switched_from) |
| prev_class->switched_from(rq, p); |
| p->sched_class->switched_to(rq, p); |
| } else if (oldprio != p->prio) |
| p->sched_class->prio_changed(rq, p, oldprio); |
| } |
| |
| static void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags) |
| { |
| const struct sched_class *class; |
| |
| if (p->sched_class == rq->curr->sched_class) { |
| rq->curr->sched_class->check_preempt_curr(rq, p, flags); |
| } else { |
| for_each_class(class) { |
| if (class == rq->curr->sched_class) |
| break; |
| if (class == p->sched_class) { |
| resched_task(rq->curr); |
| break; |
| } |
| } |
| } |
| |
| /* |
| * A queue event has occurred, and we're going to schedule. In |
| * this case, we can save a useless back to back clock update. |
| */ |
| if (rq->curr->on_rq && test_tsk_need_resched(rq->curr)) |
| rq->skip_clock_update = 1; |
| } |
| |
| #ifdef CONFIG_SMP |
| /* |
| * Is this task likely cache-hot: |
| */ |
| static int |
| task_hot(struct task_struct *p, u64 now, struct sched_domain *sd) |
| { |
| s64 delta; |
| |
| if (p->sched_class != &fair_sched_class) |
| return 0; |
| |
| if (unlikely(p->policy == SCHED_IDLE)) |
| return 0; |
| |
| /* |
| * Buddy candidates are cache hot: |
| */ |
| if (sched_feat(CACHE_HOT_BUDDY) && this_rq()->nr_running && |
| (&p->se == cfs_rq_of(&p->se)->next || |
| &p->se == cfs_rq_of(&p->se)->last)) |
| return 1; |
| |
| if (sysctl_sched_migration_cost == -1) |
| return 1; |
| if (sysctl_sched_migration_cost == 0) |
| return 0; |
| |
| delta = now - p->se.exec_start; |
| |
| return delta < (s64)sysctl_sched_migration_cost; |
| } |
| |
| void set_task_cpu(struct task_struct *p, unsigned int new_cpu) |
| { |
| #ifdef CONFIG_SCHED_DEBUG |
| /* |
| * We should never call set_task_cpu() on a blocked task, |
| * ttwu() will sort out the placement. |
| */ |
| WARN_ON_ONCE(p->state != TASK_RUNNING && p->state != TASK_WAKING && |
| !(task_thread_info(p)->preempt_count & PREEMPT_ACTIVE)); |
| |
| #ifdef CONFIG_LOCKDEP |
| /* |
| * The caller should hold either p->pi_lock or rq->lock, when changing |
| * a task's CPU. ->pi_lock for waking tasks, rq->lock for runnable tasks. |
| * |
| * sched_move_task() holds both and thus holding either pins the cgroup, |
| * see set_task_rq(). |
| * |
| * Furthermore, all task_rq users should acquire both locks, see |
| * task_rq_lock(). |
| */ |
| WARN_ON_ONCE(debug_locks && !(lockdep_is_held(&p->pi_lock) || |
| lockdep_is_held(&task_rq(p)->lock))); |
| #endif |
| #endif |
| |
| trace_sched_migrate_task(p, new_cpu); |
| |
| if (task_cpu(p) != new_cpu) { |
| p->se.nr_migrations++; |
| perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS, 1, NULL, 0); |
| } |
| |
| __set_task_cpu(p, new_cpu); |
| } |
| |
| struct migration_arg { |
| struct task_struct *task; |
| int dest_cpu; |
| }; |
| |
| static int migration_cpu_stop(void *data); |
| |
| /* |
| * wait_task_inactive - wait for a thread to unschedule. |
| * |
| * If @match_state is nonzero, it's the @p->state value just checked and |
| * not expected to change. If it changes, i.e. @p might have woken up, |
| * then return zero. When we succeed in waiting for @p to be off its CPU, |
| * we return a positive number (its total switch count). If a second call |
| * a short while later returns the same number, the caller can be sure that |
| * @p has remained unscheduled the whole time. |
| * |
| * The caller must ensure that the task *will* unschedule sometime soon, |
| * else this function might spin for a *long* time. This function can't |
| * be called with interrupts off, or it may introduce deadlock with |
| * smp_call_function() if an IPI is sent by the same process we are |
| * waiting to become inactive. |
| */ |
| unsigned long wait_task_inactive(struct task_struct *p, long match_state) |
| { |
| unsigned long flags; |
| int running, on_rq; |
| unsigned long ncsw; |
| struct rq *rq; |
| |
| for (;;) { |
| /* |
| * We do the initial early heuristics without holding |
| * any task-queue locks at all. We'll only try to get |
| * the runqueue lock when things look like they will |
| * work out! |
| */ |
| rq = task_rq(p); |
| |
| /* |
| * If the task is actively running on another CPU |
| * still, just relax and busy-wait without holding |
| * any locks. |
| * |
| * NOTE! Since we don't hold any locks, it's not |
| * even sure that "rq" stays as the right runqueue! |
| * But we don't care, since "task_running()" will |
| * return false if the runqueue has changed and p |
| * is actually now running somewhere else! |
| */ |
| while (task_running(rq, p)) { |
| if (match_state && unlikely(p->state != match_state)) |
| return 0; |
| cpu_relax(); |
| } |
| |
| /* |
| * Ok, time to look more closely! We need the rq |
| * lock now, to be *sure*. If we're wrong, we'll |
| * just go back and repeat. |
| */ |
| rq = task_rq_lock(p, &flags); |
| trace_sched_wait_task(p); |
| running = task_running(rq, p); |
| on_rq = p->on_rq; |
| ncsw = 0; |
| if (!match_state || p->state == match_state) |
| ncsw = p->nvcsw | LONG_MIN; /* sets MSB */ |
| task_rq_unlock(rq, p, &flags); |
| |
| /* |
| * If it changed from the expected state, bail out now. |
| */ |
| if (unlikely(!ncsw)) |
| break; |
| |
| /* |
| * Was it really running after all now that we |
| * checked with the proper locks actually held? |
| * |
| * Oops. Go back and try again.. |
| */ |
| if (unlikely(running)) { |
| cpu_relax(); |
| continue; |
| } |
| |
| /* |
| * It's not enough that it's not actively running, |
| * it must be off the runqueue _entirely_, and not |
| * preempted! |
| * |
| * So if it was still runnable (but just not actively |
| * running right now), it's preempted, and we should |
| * yield - it could be a while. |
| */ |
| if (unlikely(on_rq)) { |
| ktime_t to = ktime_set(0, NSEC_PER_SEC/HZ); |
| |
| set_current_state(TASK_UNINTERRUPTIBLE); |
| schedule_hrtimeout(&to, HRTIMER_MODE_REL); |
| continue; |
| } |
| |
| /* |
| * Ahh, all good. It wasn't running, and it wasn't |
| * runnable, which means that it will never become |
| * running in the future either. We're all done! |
| */ |
| break; |
| } |
| |
| return ncsw; |
| } |
| |
| /*** |
| * kick_process - kick a running thread to enter/exit the kernel |
| * @p: the to-be-kicked thread |
| * |
| * Cause a process which is running on another CPU to enter |
| * kernel-mode, without any delay. (to get signals handled.) |
| * |
| * NOTE: this function doesn't have to take the runqueue lock, |
| * because all it wants to ensure is that the remote task enters |
| * the kernel. If the IPI races and the task has been migrated |
| * to another CPU then no harm is done and the purpose has been |
| * achieved as well. |
| */ |
| void kick_process(struct task_struct *p) |
| { |
| int cpu; |
| |
| preempt_disable(); |
| cpu = task_cpu(p); |
| if ((cpu != smp_processor_id()) && task_curr(p)) |
| smp_send_reschedule(cpu); |
| preempt_enable(); |
| } |
| EXPORT_SYMBOL_GPL(kick_process); |
| #endif /* CONFIG_SMP */ |
| |
| #ifdef CONFIG_SMP |
| /* |
| * ->cpus_allowed is protected by both rq->lock and p->pi_lock |
| */ |
| static int select_fallback_rq(int cpu, struct task_struct *p) |
| { |
| int dest_cpu; |
| const struct cpumask *nodemask = cpumask_of_node(cpu_to_node(cpu)); |
| |
| /* Look for allowed, online CPU in same node. */ |
| for_each_cpu_and(dest_cpu, nodemask, cpu_active_mask) |
| if (cpumask_test_cpu(dest_cpu, tsk_cpus_allowed(p))) |
| return dest_cpu; |
| |
| /* Any allowed, online CPU? */ |
| dest_cpu = cpumask_any_and(tsk_cpus_allowed(p), cpu_active_mask); |
| if (dest_cpu < nr_cpu_ids) |
| return dest_cpu; |
| |
| /* No more Mr. Nice Guy. */ |
| dest_cpu = cpuset_cpus_allowed_fallback(p); |
| /* |
| * Don't tell them about moving exiting tasks or |
| * kernel threads (both mm NULL), since they never |
| * leave kernel. |
| */ |
| if (p->mm && printk_ratelimit()) { |
| printk(KERN_INFO "process %d (%s) no longer affine to cpu%d\n", |
| task_pid_nr(p), p->comm, cpu); |
| } |
| |
| return dest_cpu; |
| } |
| |
| /* |
| * The caller (fork, wakeup) owns p->pi_lock, ->cpus_allowed is stable. |
| */ |
| static inline |
| int select_task_rq(struct task_struct *p, int sd_flags, int wake_flags) |
| { |
| int cpu = p->sched_class->select_task_rq(p, sd_flags, wake_flags); |
| |
| /* |
| * In order not to call set_task_cpu() on a blocking task we need |
| * to rely on ttwu() to place the task on a valid ->cpus_allowed |
| * cpu. |
| * |
| * Since this is common to all placement strategies, this lives here. |
| * |
| * [ this allows ->select_task() to simply return task_cpu(p) and |
| * not worry about this generic constraint ] |
| */ |
| if (unlikely(!cpumask_test_cpu(cpu, tsk_cpus_allowed(p)) || |
| !cpu_online(cpu))) |
| cpu = select_fallback_rq(task_cpu(p), p); |
| |
| return cpu; |
| } |
| |
| static void update_avg(u64 *avg, u64 sample) |
| { |
| s64 diff = sample - *avg; |
| *avg += diff >> 3; |
| } |
| #endif |
| |
| static void |
| ttwu_stat(struct task_struct *p, int cpu, int wake_flags) |
| { |
| #ifdef CONFIG_SCHEDSTATS |
| struct rq *rq = this_rq(); |
| |
| #ifdef CONFIG_SMP |
| int this_cpu = smp_processor_id(); |
| |
| if (cpu == this_cpu) { |
| schedstat_inc(rq, ttwu_local); |
| schedstat_inc(p, se.statistics.nr_wakeups_local); |
| } else { |
| struct sched_domain *sd; |
| |
| schedstat_inc(p, se.statistics.nr_wakeups_remote); |
| rcu_read_lock(); |
| for_each_domain(this_cpu, sd) { |
| if (cpumask_test_cpu(cpu, sched_domain_span(sd))) { |
| schedstat_inc(sd, ttwu_wake_remote); |
| break; |
| } |
| } |
| rcu_read_unlock(); |
| } |
| |
| if (wake_flags & WF_MIGRATED) |
| schedstat_inc(p, se.statistics.nr_wakeups_migrate); |
| |
| #endif /* CONFIG_SMP */ |
| |
| schedstat_inc(rq, ttwu_count); |
| schedstat_inc(p, se.statistics.nr_wakeups); |
| |
| if (wake_flags & WF_SYNC) |
| schedstat_inc(p, se.statistics.nr_wakeups_sync); |
| |
| #endif /* CONFIG_SCHEDSTATS */ |
| } |
| |
| static void ttwu_activate(struct rq *rq, struct task_struct *p, int en_flags) |
| { |
| activate_task(rq, p, en_flags); |
| p->on_rq = 1; |
| |
| /* if a worker is waking up, notify workqueue */ |
| if (p->flags & PF_WQ_WORKER) |
| wq_worker_waking_up(p, cpu_of(rq)); |
| } |
| |
| /* |
| * Mark the task runnable and perform wakeup-preemption. |
| */ |
| static void |
| ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags) |
| { |
| trace_sched_wakeup(p, true); |
| check_preempt_curr(rq, p, wake_flags); |
| |
| p->state = TASK_RUNNING; |
| #ifdef CONFIG_SMP |
| if (p->sched_class->task_woken) |
| p->sched_class->task_woken(rq, p); |
| |
| if (rq->idle_stamp) { |
| u64 delta = rq->clock - rq->idle_stamp; |
| u64 max = 2*sysctl_sched_migration_cost; |
| |
| if (delta > max) |
| rq->avg_idle = max; |
| else |
| update_avg(&rq->avg_idle, delta); |
| rq->idle_stamp = 0; |
| } |
| #endif |
| } |
| |
| static void |
| ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags) |
| { |
| #ifdef CONFIG_SMP |
| if (p->sched_contributes_to_load) |
| rq->nr_uninterruptible--; |
| #endif |
| |
| ttwu_activate(rq, p, ENQUEUE_WAKEUP | ENQUEUE_WAKING); |
| ttwu_do_wakeup(rq, p, wake_flags); |
| } |
| |
| /* |
| * Called in case the task @p isn't fully descheduled from its runqueue, |
| * in this case we must do a remote wakeup. Its a 'light' wakeup though, |
| * since all we need to do is flip p->state to TASK_RUNNING, since |
| * the task is still ->on_rq. |
| */ |
| static int ttwu_remote(struct task_struct *p, int wake_flags) |
| { |
| struct rq *rq; |
| int ret = 0; |
| |
| rq = __task_rq_lock(p); |
| if (p->on_rq) { |
| ttwu_do_wakeup(rq, p, wake_flags); |
| ret = 1; |
| } |
| __task_rq_unlock(rq); |
| |
| return ret; |
| } |
| |
| #ifdef CONFIG_SMP |
| static void sched_ttwu_pending(void) |
| { |
| struct rq *rq = this_rq(); |
| struct llist_node *llist = llist_del_all(&rq->wake_list); |
| struct task_struct *p; |
| |
| raw_spin_lock(&rq->lock); |
| |
| while (llist) { |
| p = llist_entry(llist, struct task_struct, wake_entry); |
| llist = llist_next(llist); |
| ttwu_do_activate(rq, p, 0); |
| } |
| |
| raw_spin_unlock(&rq->lock); |
| } |
| |
| void scheduler_ipi(void) |
| { |
| if (llist_empty(&this_rq()->wake_list) && !got_nohz_idle_kick()) |
| return; |
| |
| /* |
| * Not all reschedule IPI handlers call irq_enter/irq_exit, since |
| * traditionally all their work was done from the interrupt return |
| * path. Now that we actually do some work, we need to make sure |
| * we do call them. |
| * |
| * Some archs already do call them, luckily irq_enter/exit nest |
| * properly. |
| * |
| * Arguably we should visit all archs and update all handlers, |
| * however a fair share of IPIs are still resched only so this would |
| * somewhat pessimize the simple resched case. |
| */ |
| irq_enter(); |
| sched_ttwu_pending(); |
| |
| /* |
| * Check if someone kicked us for doing the nohz idle load balance. |
| */ |
| if (unlikely(got_nohz_idle_kick() && !need_resched())) { |
| this_rq()->idle_balance = 1; |
| raise_softirq_irqoff(SCHED_SOFTIRQ); |
| } |
| irq_exit(); |
| } |
| |
| static void ttwu_queue_remote(struct task_struct *p, int cpu) |
| { |
| if (llist_add(&p->wake_entry, &cpu_rq(cpu)->wake_list)) |
| smp_send_reschedule(cpu); |
| } |
| |
| #ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW |
| static int ttwu_activate_remote(struct task_struct *p, int wake_flags) |
| { |
| struct rq *rq; |
| int ret = 0; |
| |
| rq = __task_rq_lock(p); |
| if (p->on_cpu) { |
| ttwu_activate(rq, p, ENQUEUE_WAKEUP); |
| ttwu_do_wakeup(rq, p, wake_flags); |
| ret = 1; |
| } |
| __task_rq_unlock(rq); |
| |
| return ret; |
| |
| } |
| #endif /* __ARCH_WANT_INTERRUPTS_ON_CTXSW */ |
| #endif /* CONFIG_SMP */ |
| |
| static void ttwu_queue(struct task_struct *p, int cpu) |
| { |
| struct rq *rq = cpu_rq(cpu); |
| |
| #if defined(CONFIG_SMP) |
| if (sched_feat(TTWU_QUEUE) && cpu != smp_processor_id()) { |
| sched_clock_cpu(cpu); /* sync clocks x-cpu */ |
| ttwu_queue_remote(p, cpu); |
| return; |
| } |
| #endif |
| |
| raw_spin_lock(&rq->lock); |
| ttwu_do_activate(rq, p, 0); |
| raw_spin_unlock(&rq->lock); |
| } |
| |
| /** |
| * try_to_wake_up - wake up a thread |
| * @p: the thread to be awakened |
| * @state: the mask of task states that can be woken |
| * @wake_flags: wake modifier flags (WF_*) |
| * |
| * Put it on the run-queue if it's not already there. The "current" |
| * thread is always on the run-queue (except when the actual |
| * re-schedule is in progress), and as such you're allowed to do |
| * the simpler "current->state = TASK_RUNNING" to mark yourself |
| * runnable without the overhead of this. |
| * |
| * Returns %true if @p was woken up, %false if it was already running |
| * or @state didn't match @p's state. |
| */ |
| static int |
| try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) |
| { |
| unsigned long flags; |
| int cpu, success = 0; |
| |
| smp_wmb(); |
| raw_spin_lock_irqsave(&p->pi_lock, flags); |
| if (!(p->state & state)) |
| goto out; |
| |
| success = 1; /* we're going to change ->state */ |
| cpu = task_cpu(p); |
| |
| if (p->on_rq && ttwu_remote(p, wake_flags)) |
| goto stat; |
| |
| #ifdef CONFIG_SMP |
| /* |
| * If the owning (remote) cpu is still in the middle of schedule() with |
| * this task as prev, wait until its done referencing the task. |
| */ |
| while (p->on_cpu) { |
| #ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW |
| /* |
| * In case the architecture enables interrupts in |
| * context_switch(), we cannot busy wait, since that |
| * would lead to deadlocks when an interrupt hits and |
| * tries to wake up @prev. So bail and do a complete |
| * remote wakeup. |
| */ |
| if (ttwu_activate_remote(p, wake_flags)) |
| goto stat; |
| #else |
| cpu_relax(); |
| #endif |
| } |
| /* |
| * Pairs with the smp_wmb() in finish_lock_switch(). |
| */ |
| smp_rmb(); |
| |
| p->sched_contributes_to_load = !!task_contributes_to_load(p); |
| p->state = TASK_WAKING; |
| |
| if (p->sched_class->task_waking) |
| p->sched_class->task_waking(p); |
| |
| cpu = select_task_rq(p, SD_BALANCE_WAKE, wake_flags); |
| if (task_cpu(p) != cpu) { |
| wake_flags |= WF_MIGRATED; |
| set_task_cpu(p, cpu); |
| } |
| #endif /* CONFIG_SMP */ |
| |
| ttwu_queue(p, cpu); |
| stat: |
| ttwu_stat(p, cpu, wake_flags); |
| out: |
| raw_spin_unlock_irqrestore(&p->pi_lock, flags); |
| |
| return success; |
| } |
| |
| /** |
| * try_to_wake_up_local - try to wake up a local task with rq lock held |
| * @p: the thread to be awakened |
| * |
| * Put @p on the run-queue if it's not already there. The caller must |
| * ensure that this_rq() is locked, @p is bound to this_rq() and not |
| * the current task. |
| */ |
| static void try_to_wake_up_local(struct task_struct *p) |
| { |
| struct rq *rq = task_rq(p); |
| |
| BUG_ON(rq != this_rq()); |
| BUG_ON(p == current); |
| lockdep_assert_held(&rq->lock); |
| |
| if (!raw_spin_trylock(&p->pi_lock)) { |
| raw_spin_unlock(&rq->lock); |
| raw_spin_lock(&p->pi_lock); |
| raw_spin_lock(&rq->lock); |
| } |
| |
| if (!(p->state & TASK_NORMAL)) |
| goto out; |
| |
| if (!p->on_rq) |
| ttwu_activate(rq, p, ENQUEUE_WAKEUP); |
| |
| ttwu_do_wakeup(rq, p, 0); |
| ttwu_stat(p, smp_processor_id(), 0); |
| out: |
| raw_spin_unlock(&p->pi_lock); |
| } |
| |
| /** |
| * wake_up_process - Wake up a specific process |
| * @p: The process to be woken up. |
| * |
| * Attempt to wake up the nominated process and move it to the set of runnable |
| * processes. Returns 1 if the process was woken up, 0 if it was already |
| * running. |
| * |
| * It may be assumed that this function implies a write memory barrier before |
| * changing the task state if and only if any tasks are woken up. |
| */ |
| int wake_up_process(struct task_struct *p) |
| { |
| return try_to_wake_up(p, TASK_ALL, 0); |
| } |
| EXPORT_SYMBOL(wake_up_process); |
| |
| int wake_up_state(struct task_struct *p, unsigned int state) |
| { |
| return try_to_wake_up(p, state, 0); |
| } |
| |
| /* |
| * Perform scheduler related setup for a newly forked process p. |
| * p is forked by current. |
| * |
| * __sched_fork() is basic setup used by init_idle() too: |
| */ |
| static void __sched_fork(struct task_struct *p) |
| { |
| p->on_rq = 0; |
| |
| p->se.on_rq = 0; |
| p->se.exec_start = 0; |
| p->se.sum_exec_runtime = 0; |
| p->se.prev_sum_exec_runtime = 0; |
| p->se.nr_migrations = 0; |
| p->se.vruntime = 0; |
| INIT_LIST_HEAD(&p->se.group_node); |
| |
| #ifdef CONFIG_SCHEDSTATS |
| memset(&p->se.statistics, 0, sizeof(p->se.statistics)); |
| #endif |
| |
| INIT_LIST_HEAD(&p->rt.run_list); |
| |
| #ifdef CONFIG_PREEMPT_NOTIFIERS |
| INIT_HLIST_HEAD(&p->preempt_notifiers); |
| #endif |
| } |
| |
| /* |
| * fork()/clone()-time setup: |
| */ |
| void sched_fork(struct task_struct *p) |
| { |
| unsigned long flags; |
| int cpu = get_cpu(); |
| |
| __sched_fork(p); |
| /* |
| * We mark the process as running here. This guarantees that |
| * nobody will actually run it, and a signal or other external |
| * event cannot wake it up and insert it on the runqueue either. |
| */ |
| p->state = TASK_RUNNING; |
| |
| /* |
| * Make sure we do not leak PI boosting priority to the child. |
| */ |
| p->prio = current->normal_prio; |
| |
| /* |
| * Revert to default priority/policy on fork if requested. |
| */ |
| if (unlikely(p->sched_reset_on_fork)) { |
| if (task_has_rt_policy(p)) { |
| p->policy = SCHED_NORMAL; |
| p->static_prio = NICE_TO_PRIO(0); |
| p->rt_priority = 0; |
| } else if (PRIO_TO_NICE(p->static_prio) < 0) |
| p->static_prio = NICE_TO_PRIO(0); |
| |
| p->prio = p->normal_prio = __normal_prio(p); |
| set_load_weight(p); |
| |
| /* |
| * We don't need the reset flag anymore after the fork. It has |
| * fulfilled its duty: |
| */ |
| p->sched_reset_on_fork = 0; |
| } |
| |
| if (!rt_prio(p->prio)) |
| p->sched_class = &fair_sched_class; |
| |
| if (p->sched_class->task_fork) |
| p->sched_class->task_fork(p); |
| |
| /* |
| * The child is not yet in the pid-hash so no cgroup attach races, |
| * and the cgroup is pinned to this child due to cgroup_fork() |
| * is ran before sched_fork(). |
| * |
| * Silence PROVE_RCU. |
| */ |
| raw_spin_lock_irqsave(&p->pi_lock, flags); |
| set_task_cpu(p, cpu); |
| raw_spin_unlock_irqrestore(&p->pi_lock, flags); |
| |
| #if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT) |
| if (likely(sched_info_on())) |
| memset(&p->sched_info, 0, sizeof(p->sched_info)); |
| #endif |
| #if defined(CONFIG_SMP) |
| p->on_cpu = 0; |
| #endif |
| #ifdef CONFIG_PREEMPT_COUNT |
| /* Want to start with kernel preemption disabled. */ |
| task_thread_info(p)->preempt_count = 1; |
| #endif |
| #ifdef CONFIG_SMP |
| plist_node_init(&p->pushable_tasks, MAX_PRIO); |
| #endif |
| |
| put_cpu(); |
| } |
| |
| /* |
| * wake_up_new_task - wake up a newly created task for the first time. |
| * |
| * This function will do some initial scheduler statistics housekeeping |
| * that must be done for every newly created context, then puts the task |
| * on the runqueue and wakes it. |
| */ |
| void wake_up_new_task(struct task_struct *p) |
| { |
| unsigned long flags; |
| struct rq *rq; |
| |
| raw_spin_lock_irqsave(&p->pi_lock, flags); |
| #ifdef CONFIG_SMP |
| /* |
| * Fork balancing, do it here and not earlier because: |
| * - cpus_allowed can change in the fork path |
| * - any previously selected cpu might disappear through hotplug |
| */ |
| set_task_cpu(p, select_task_rq(p, SD_BALANCE_FORK, 0)); |
| #endif |
| |
| rq = __task_rq_lock(p); |
| activate_task(rq, p, 0); |
| p->on_rq = 1; |
| trace_sched_wakeup_new(p, true); |
| check_preempt_curr(rq, p, WF_FORK); |
| #ifdef CONFIG_SMP |
| if (p->sched_class->task_woken) |
| p->sched_class->task_woken(rq, p); |
| #endif |
| task_rq_unlock(rq, p, &flags); |
| } |
| |
| #ifdef CONFIG_PREEMPT_NOTIFIERS |
| |
| /** |
| * preempt_notifier_register - tell me when current is being preempted & rescheduled |
| * @notifier: notifier struct to register |
| */ |
| void preempt_notifier_register(struct preempt_notifier *notifier) |
| { |
| hlist_add_head(¬ifier->link, ¤t->preempt_notifiers); |
| } |
| EXPORT_SYMBOL_GPL(preempt_notifier_register); |
| |
| /** |
| * preempt_notifier_unregister - no longer interested in preemption notifications |
| * @notifier: notifier struct to unregister |
| * |
| * This is safe to call from within a preemption notifier. |
| */ |
| void preempt_notifier_unregister(struct preempt_notifier *notifier) |
| { |
| hlist_del(¬ifier->link); |
| } |
| EXPORT_SYMBOL_GPL(preempt_notifier_unregister); |
| |
| static void fire_sched_in_preempt_notifiers(struct task_struct *curr) |
| { |
| struct preempt_notifier *notifier; |
| struct hlist_node *node; |
| |
| hlist_for_each_entry(notifier, node, &curr->preempt_notifiers, link) |
| notifier->ops->sched_in(notifier, raw_smp_processor_id()); |
| } |
| |
| static void |
| fire_sched_out_preempt_notifiers(struct task_struct *curr, |
| struct task_struct *next) |
| { |
| struct preempt_notifier *notifier; |
| struct hlist_node *node; |
| |
| hlist_for_each_entry(notifier, node, &curr->preempt_notifiers, link) |
| notifier->ops->sched_out(notifier, next); |
| } |
| |
| #else /* !CONFIG_PREEMPT_NOTIFIERS */ |
| |
| static void fire_sched_in_preempt_notifiers(struct task_struct *curr) |
| { |
| } |
| |
| static void |
| fire_sched_out_preempt_notifiers(struct task_struct *curr, |
| struct task_struct *next) |
| { |
| } |
| |
| #endif /* CONFIG_PREEMPT_NOTIFIERS */ |
| |
| /** |
| * prepare_task_switch - prepare to switch tasks |
| * @rq: the runqueue preparing to switch |
| * @prev: the current task that is being switched out |
| * @next: the task we are going to switch to. |
| * |
| * This is called with the rq lock held and interrupts off. It must |
| * be paired with a subsequent finish_task_switch after the context |
| * switch. |
| * |
| * prepare_task_switch sets up locking and calls architecture specific |
| * hooks. |
| */ |
| static inline void |
| prepare_task_switch(struct rq *rq, struct task_struct *prev, |
| struct task_struct *next) |
| { |
| sched_info_switch(prev, next); |
| perf_event_task_sched_out(prev, next); |
| fire_sched_out_preempt_notifiers(prev, next); |
| prepare_lock_switch(rq, next); |
| prepare_arch_switch(next); |
| trace_sched_switch(prev, next); |
| } |
| |
| /** |
| * finish_task_switch - clean up after a task-switch |
| * @rq: runqueue associated with task-switch |
| * @prev: the thread we just switched away from. |
| * |
| * finish_task_switch must be called after the context switch, paired |
| * with a prepare_task_switch call before the context switch. |
| * finish_task_switch will reconcile locking set up by prepare_task_switch, |
| * and do any other architecture-specific cleanup actions. |
| * |
| * Note that we may have delayed dropping an mm in context_switch(). If |
| * so, we finish that here outside of the runqueue lock. (Doing it |
| * with the lock held can cause deadlocks; see schedule() for |
| * details.) |
| */ |
| static void finish_task_switch(struct rq *rq, struct task_struct *prev) |
| __releases(rq->lock) |
| { |
| struct mm_struct *mm = rq->prev_mm; |
| long prev_state; |
| |
| rq->prev_mm = NULL; |
| |
| /* |
| * A task struct has one reference for the use as "current". |
| * If a task dies, then it sets TASK_DEAD in tsk->state and calls |
| * schedule one last time. The schedule call will never return, and |
| * the scheduled task must drop that reference. |
| * The test for TASK_DEAD must occur while the runqueue locks are |
| * still held, otherwise prev could be scheduled on another cpu, die |
| * there before we look at prev->state, and then the reference would |
| * be dropped twice. |
| * Manfred Spraul <manfred@colorfullife.com> |
| */ |
| prev_state = prev->state; |
| finish_arch_switch(prev); |
| #ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW |
| local_irq_disable(); |
| #endif /* __ARCH_WANT_INTERRUPTS_ON_CTXSW */ |
| perf_event_task_sched_in(prev, current); |
| #ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW |
| local_irq_enable(); |
| #endif /* __ARCH_WANT_INTERRUPTS_ON_CTXSW */ |
| finish_lock_switch(rq, prev); |
| |
| fire_sched_in_preempt_notifiers(current); |
| if (mm) |
| mmdrop(mm); |
| if (unlikely(prev_state == TASK_DEAD)) { |
| /* |
| * Remove function-return probe instances associated with this |
| * task and put them back on the free list. |
| */ |
| kprobe_flush_task(prev); |
| put_task_struct(prev); |
| } |
| } |
| |
| #ifdef CONFIG_SMP |
| |
| /* assumes rq->lock is held */ |
| static inline void pre_schedule(struct rq *rq, struct task_struct *prev) |
| { |
| if (prev->sched_class->pre_schedule) |
| prev->sched_class->pre_schedule(rq, prev); |
| } |
| |
| /* rq->lock is NOT held, but preemption is disabled */ |
| static inline void post_schedule(struct rq *rq) |
| { |
| if (rq->post_schedule) { |
| unsigned long flags; |
| |
| raw_spin_lock_irqsave(&rq->lock, flags); |
| if (rq->curr->sched_class->post_schedule) |
| rq->curr->sched_class->post_schedule(rq); |
| raw_spin_unlock_irqrestore(&rq->lock, flags); |
| |
| rq->post_schedule = 0; |
| } |
| } |
| |
| #else |
| |
| static inline void pre_schedule(struct rq *rq, struct task_struct *p) |
| { |
| } |
| |
| static inline void post_schedule(struct rq *rq) |
| { |
| } |
| |
| #endif |
| |
| /** |
| * schedule_tail - first thing a freshly forked thread must call. |
| * @prev: the thread we just switched away from. |
| */ |
| asmlinkage void schedule_tail(struct task_struct *prev) |
| __releases(rq->lock) |
| { |
| struct rq *rq = this_rq(); |
| |
| finish_task_switch(rq, prev); |
| |
| /* |
| * FIXME: do we need to worry about rq being invalidated by the |
| * task_switch? |
| */ |
| post_schedule(rq); |
| |
| #ifdef __ARCH_WANT_UNLOCKED_CTXSW |
| /* In this case, finish_task_switch does not reenable preemption */ |
| preempt_enable(); |
| #endif |
| if (current->set_child_tid) |
| put_user(task_pid_vnr(current), current->set_child_tid); |
| } |
| |
| /* |
| * context_switch - switch to the new MM and the new |
| * thread's register state. |
| */ |
| static inline void |
| context_switch(struct rq *rq, struct task_struct *prev, |
| struct task_struct *next) |
| { |
| struct mm_struct *mm, *oldmm; |
| |
| prepare_task_switch(rq, prev, next); |
| |
| mm = next->mm; |
| oldmm = prev->active_mm; |
| /* |
| * For paravirt, this is coupled with an exit in switch_to to |
| * combine the page table reload and the switch backend into |
| * one hypercall. |
| */ |
| arch_start_context_switch(prev); |
| |
| if (!mm) { |
| next->active_mm = oldmm; |
| atomic_inc(&oldmm->mm_count); |
| enter_lazy_tlb(oldmm, next); |
| } else |
| switch_mm(oldmm, mm, next); |
| |
| if (!prev->mm) { |
| prev->active_mm = NULL; |
| rq->prev_mm = oldmm; |
| } |
| /* |
| * Since the runqueue lock will be released by the next |
| * task (which is an invalid locking op but in the case |
| * of the scheduler it's an obvious special-case), so we |
| * do an early lockdep release here: |
| */ |
| #ifndef __ARCH_WANT_UNLOCKED_CTXSW |
| spin_release(&rq->lock.dep_map, 1, _THIS_IP_); |
| #endif |
| |
| /* Here we just switch the register state and the stack. */ |
| switch_to(prev, next, prev); |
| |
| barrier(); |
| /* |
| * this_rq must be evaluated again because prev may have moved |
| * CPUs since it called schedule(), thus the 'rq' on its stack |
| * frame will be invalid. |
| */ |
| finish_task_switch(this_rq(), prev); |
| } |
| |
| /* |
| * nr_running, nr_uninterruptible and nr_context_switches: |
| * |
| * externally visible scheduler statistics: current number of runnable |
| * threads, current number of uninterruptible-sleeping threads, total |
| * number of context switches performed since bootup. |
| */ |
| unsigned long nr_running(void) |
| { |
| unsigned long i, sum = 0; |
| |
| for_each_online_cpu(i) |
| sum += cpu_rq(i)->nr_running; |
| |
| return sum; |
| } |
| |
| unsigned long nr_uninterruptible(void) |
| { |
| unsigned long i, sum = 0; |
| |
| for_each_possible_cpu(i) |
| sum += cpu_rq(i)->nr_uninterruptible; |
| |
| /* |
| * Since we read the counters lockless, it might be slightly |
| * inaccurate. Do not allow it to go below zero though: |
| */ |
| if (unlikely((long)sum < 0)) |
| sum = 0; |
| |
| return sum; |
| } |
| |
| unsigned long long nr_context_switches(void) |
| { |
| int i; |
| unsigned long long sum = 0; |
| |
| for_each_possible_cpu(i) |
| sum += cpu_rq(i)->nr_switches; |
| |
| return sum; |
| } |
| |
| unsigned long nr_iowait(void) |
| { |
| unsigned long i, sum = 0; |
| |
| for_each_possible_cpu(i) |
| sum += atomic_read(&cpu_rq(i)->nr_iowait); |
| |
| return sum; |
| } |
| |
| unsigned long nr_iowait_cpu(int cpu) |
| { |
| struct rq *this = cpu_rq(cpu); |
| return atomic_read(&this->nr_iowait); |
| } |
| |
| unsigned long this_cpu_load(void) |
| { |
| struct rq *this = this_rq(); |
| return this->cpu_load[0]; |
| } |
| |
| |
| /* |
| * Global load-average calculations |
| * |
| * We take a distributed and async approach to calculating the global load-avg |
| * in order to minimize overhead. |
| * |
| * The global load average is an exponentially decaying average of nr_running + |
| * nr_uninterruptible. |
| * |
| * Once every LOAD_FREQ: |
| * |
| * nr_active = 0; |
| * for_each_possible_cpu(cpu) |
| * nr_active += cpu_of(cpu)->nr_running + cpu_of(cpu)->nr_uninterruptible; |
| * |
| * avenrun[n] = avenrun[0] * exp_n + nr_active * (1 - exp_n) |
| * |
| * Due to a number of reasons the above turns in the mess below: |
| * |
| * - for_each_possible_cpu() is prohibitively expensive on machines with |
| * serious number of cpus, therefore we need to take a distributed approach |
| * to calculating nr_active. |
| * |
| * \Sum_i x_i(t) = \Sum_i x_i(t) - x_i(t_0) | x_i(t_0) := 0 |
| * = \Sum_i { \Sum_j=1 x_i(t_j) - x_i(t_j-1) } |
| * |
| * So assuming nr_active := 0 when we start out -- true per definition, we |
| * can simply take per-cpu deltas and fold those into a global accumulate |
| * to obtain the same result. See calc_load_fold_active(). |
| * |
| * Furthermore, in order to avoid synchronizing all per-cpu delta folding |
| * across the machine, we assume 10 ticks is sufficient time for every |
| * cpu to have completed this task. |
| * |
| * This places an upper-bound on the IRQ-off latency of the machine. Then |
| * again, being late doesn't loose the delta, just wrecks the sample. |
| * |
| * - cpu_rq()->nr_uninterruptible isn't accurately tracked per-cpu because |
| * this would add another cross-cpu cacheline miss and atomic operation |
| * to the wakeup path. Instead we increment on whatever cpu the task ran |
| * when it went into uninterruptible state and decrement on whatever cpu |
| * did the wakeup. This means that only the sum of nr_uninterruptible over |
| * all cpus yields the correc
|