[linux-yocto] [PATCH 44/48] Revert "sched: fixup migrate disable (all tasks were bound to CPU0)"
Daniel Dragomir
daniel.dragomir at windriver.com
Mon Dec 11 05:14:14 PST 2017
From: Daniel Bristot de Oliveira <bristot at redhat.com>
This reverts commit c3d80fe97a84a8d137b93e61b2ccbca974022d84.
There is a problem in the updated version of the
migrate_disable()/enable() implementation backported in this patch.
The problem is the following:
In the new behavior, when a task is attached to the rt runqueue, it is
checked if it either can run in more than one CPU, or if it is with
migration disable. If either check is true, the rt_rq->rt_nr_migratory
counter is not increased. The counter increases otherwise.
When the task is detached, the same check is done. If either check is
true, the rt_rq->rt_nr_migratory counter is not decreased. The counter
decreases otherwise.
One important thing is that, migrate disable/enable does not touch this
counter for tasks attached to the -rt rq. So suppose the following chain
of events.
Assumptions:
Task A is the only runnable task in A Task B runs in the CPU B
rt_nr_migratory is 0, so. Task B has RT priority
Task A can run on all CPUS. B is running
A is running
Task A runs on CFS (non-rt)
CPU A/TASK A CPU B/TASK B
A takes the rt mutex X .
A disables migration .
. B rt task tries to take the rt mutex X
. As it is locked by A:
. A inherits the rt priority of B
. A is dequeued from CFS RQ of CPU A
. A is enqueued in the RT RQ of CPU A
. As migration is disabled:
. rt_nr_migratory in A is not increased
.
A enables migration
A releases the rt mutex {
A returns to its original priority
A ask to be dequeued from RT RQ {
As migration is now enabled and it can run on all CPUS {
rt_nr_migratory should be decreased
As it is 0, rt_nr_migratory under flows.
}
}
The rt_nr_migratory value must be at least 0, and that is the cause
of the BUG().
This variable is important because it notifies if there are more than one
runnable & migratable task in the runqueue. If there are more than one
tasks, the rt_rq is set as overloaded, and this tries to migrate some
tasks. This rule is important to keep the scheduler working conserving,
that is in a system with M CPUs, the M highest priority tasks should be
running.
Signed-off-by: Daniel Bristot de Oliveira <bristot at redhat.com>
Signed-off-by: Charlie Paul <cpaul.windriver at gmail.com>
Link: https://bugzilla.redhat.com/show_bug.cgi?id=1423432
---
include/linux/sched.h | 11 +-
kernel/sched/core.c | 834 ++++++++++++++++++++++++++-------------------
kernel/sched/cpudeadline.c | 4 +-
kernel/sched/cpupri.c | 4 +-
kernel/sched/deadline.c | 30 +-
kernel/sched/rt.c | 22 +-
6 files changed, 513 insertions(+), 392 deletions(-)
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 4c12a0c..695bb72 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -3697,19 +3697,14 @@ static inline int __migrate_disabled(struct task_struct *p)
/* Future-safe accessor for struct task_struct's cpus_allowed. */
static inline const struct cpumask *tsk_cpus_allowed(struct task_struct *p)
{
- if (__migrate_disabled(p))
+#ifdef CONFIG_PREEMPT_RT_FULL
+ if (p->migrate_disable)
return cpumask_of(task_cpu(p));
+#endif
return &p->cpus_allowed;
}
-static inline int tsk_nr_cpus_allowed(struct task_struct *p)
-{
- if (__migrate_disabled(p))
- return 1;
- return p->nr_cpus_allowed;
-}
-
extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask);
extern long sched_getaffinity(pid_t pid, struct cpumask *mask);
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 00c2b63..85f8b38 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -370,16 +370,17 @@ static inline void init_rq_hrtick(struct rq *rq)
typeof(mask) _mask = (mask); \
typeof(*_ptr) _old, _val = *_ptr; \
\
- for (;;) { \
- _old = cmpxchg(_ptr, _val, _val | _mask); \
- if (_old == _val) \
- break; \
- _val = _old; \
- } \
- _old; \
+ for (;;) { \
+ _old = cmpxchg(_ptr, _val, _val | _mask); \
+ if (_old == _val) \
+ break; \
+ _val = _old; \
+ } \
+ _old; \
})
#if defined(CONFIG_SMP) && defined(TIF_POLLING_NRFLAG)
+
/*
* Atomically set TIF_NEED_RESCHED and test for TIF_POLLING_NRFLAG,
* this avoids any races wrt polling state changes and thereby avoids
@@ -416,6 +417,7 @@ static bool set_nr_if_polling(struct task_struct *p)
}
#else
+
static bool set_nr_and_not_polling(struct task_struct *p)
{
set_tsk_need_resched(p);
@@ -423,6 +425,7 @@ static bool set_nr_and_not_polling(struct task_struct *p)
}
#ifdef CONFIG_SMP
+
static bool set_nr_if_polling(struct task_struct *p)
{
return false;
@@ -511,6 +514,7 @@ void resched_curr(struct rq *rq)
}
#ifdef CONFIG_PREEMPT_LAZY
+
void resched_curr_lazy(struct rq *rq)
{
struct task_struct *curr = rq->curr;
@@ -555,6 +559,7 @@ void resched_cpu(int cpu)
#ifdef CONFIG_SMP
#ifdef CONFIG_NO_HZ_COMMON
+
/*
* In the semi idle case, use the nearest busy cpu for migrating timers
* from an idle cpu. This is good for power-savings.
@@ -575,7 +580,9 @@ int get_nohz_timer_target(void)
goto preempt_en_rt;
rcu_read_lock();
+
for_each_domain(cpu, sd) {
+
for_each_cpu(i, sched_domain_span(sd)) {
if (cpu == i)
continue;
@@ -595,6 +602,7 @@ int get_nohz_timer_target(void)
preempt_enable_rt();
return cpu;
}
+
/*
* When add_timer_on() enqueues a timer into the timer wheel of an
* idle CPU then this timer might expire before the next timer event
@@ -627,10 +635,10 @@ static bool wake_up_full_nohz_cpu(int cpu)
* empty IRQ.
*/
if (cpu_is_offline(cpu))
- return true; /* Don't try to wake offline CPUs. */
+ return true; /* Don't try to wake offline CPUs. */
if (tick_nohz_full_cpu(cpu)) {
if (cpu != smp_processor_id() ||
- tick_nohz_tick_stopped())
+ tick_nohz_tick_stopped())
tick_nohz_full_kick_cpu(cpu);
return true;
}
@@ -677,6 +685,7 @@ static inline bool got_nohz_idle_kick(void)
#endif /* CONFIG_NO_HZ_COMMON */
#ifdef CONFIG_NO_HZ_FULL
+
bool sched_can_stop_tick(struct rq *rq)
{
int fifo_nr_running;
@@ -720,7 +729,7 @@ void sched_avg_update(struct rq *rq)
{
s64 period = sched_avg_period();
- while ((s64)(rq_clock(rq) - rq->age_stamp) > period) {
+ while ((s64) (rq_clock(rq) - rq->age_stamp) > period) {
/*
* Inline assembly required to prevent the compiler
* optimising this loop into a divmod call.
@@ -736,6 +745,7 @@ void sched_avg_update(struct rq *rq)
#if defined(CONFIG_RT_GROUP_SCHED) || (defined(CONFIG_FAIR_GROUP_SCHED) && \
(defined(CONFIG_SMP) || defined(CONFIG_CFS_BANDWIDTH)))
+
/*
* Iterate task_group tree rooted at *from, calling @down when first entering a
* node and @up when leaving it for the final time.
@@ -743,7 +753,7 @@ void sched_avg_update(struct rq *rq)
* Caller must hold rcu_lock or sufficient equivalent.
*/
int walk_tg_tree_from(struct task_group *from,
- tg_visitor down, tg_visitor up, void *data)
+ tg_visitor down, tg_visitor up, void *data)
{
struct task_group *parent, *child;
int ret;
@@ -754,6 +764,7 @@ int walk_tg_tree_from(struct task_group *from,
ret = (*down)(parent, data);
if (ret)
goto out;
+
list_for_each_entry_rcu(child, &parent->children, siblings) {
parent = child;
goto down;
@@ -831,10 +842,10 @@ void deactivate_task(struct rq *rq, struct task_struct *p, int flags)
static void update_rq_clock_task(struct rq *rq, s64 delta)
{
-/*
- * In theory, the compile should just see 0 here, and optimize out the call
- * to sched_rt_avg_update. But I don't trust it...
- */
+ /*
+ * In theory, the compile should just see 0 here, and optimize out the call
+ * to sched_rt_avg_update. But I don't trust it...
+ */
#if defined(CONFIG_IRQ_TIME_ACCOUNTING) || defined(CONFIG_PARAVIRT_TIME_ACCOUNTING)
s64 steal = 0, irq_delta = 0;
#endif
@@ -885,7 +896,7 @@ static void update_rq_clock_task(struct rq *rq, s64 delta)
void sched_set_stop_task(int cpu, struct task_struct *stop)
{
- struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 };
+ struct sched_param param = {.sched_priority = MAX_RT_PRIO - 1};
struct task_struct *old_stop = cpu_rq(cpu)->stop;
if (stop) {
@@ -933,9 +944,9 @@ static inline int normal_prio(struct task_struct *p)
int prio;
if (task_has_dl_policy(p))
- prio = MAX_DL_PRIO-1;
+ prio = MAX_DL_PRIO - 1;
else if (task_has_rt_policy(p))
- prio = MAX_RT_PRIO-1 - p->rt_priority;
+ prio = MAX_RT_PRIO - 1 - p->rt_priority;
else
prio = __normal_prio(p);
return prio;
@@ -980,8 +991,8 @@ inline int task_curr(const struct task_struct *p)
* balance_callback().
*/
static inline void check_class_changed(struct rq *rq, struct task_struct *p,
- const struct sched_class *prev_class,
- int oldprio)
+ const struct sched_class *prev_class,
+ int oldprio)
{
if (prev_class != p->sched_class) {
if (prev_class->switched_from)
@@ -999,6 +1010,7 @@ void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags)
if (p->sched_class == rq->curr->sched_class) {
rq->curr->sched_class->check_preempt_curr(rq, p, flags);
} else {
+
for_each_class(class) {
if (class == rq->curr->sched_class)
break;
@@ -1260,7 +1272,7 @@ int migrate_me(void)
* call is not atomic; no spinlocks may be held.
*/
static int __set_cpus_allowed_ptr(struct task_struct *p,
- const struct cpumask *new_mask, bool check)
+ const struct cpumask *new_mask, bool check)
{
const struct cpumask *cpu_valid_mask = cpu_active_mask;
unsigned int dest_cpu;
@@ -1313,7 +1325,7 @@ static int __set_cpus_allowed_ptr(struct task_struct *p,
dest_cpu = cpumask_any_and(cpu_valid_mask, new_mask);
if (task_running(rq, p) || p->state == TASK_WAKING) {
- struct migration_arg arg = { p, dest_cpu };
+ struct migration_arg arg = {p, dest_cpu};
/* Need help from migration thread: drop lock and wait. */
task_rq_unlock(rq, p, &rf);
stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg);
@@ -1348,7 +1360,7 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
* ttwu() will sort out the placement.
*/
WARN_ON_ONCE(p->state != TASK_RUNNING && p->state != TASK_WAKING &&
- !p->on_rq);
+ !p->on_rq);
/*
* Migrating fair class task must have p->on_rq = TASK_ON_RQ_MIGRATING,
@@ -1356,8 +1368,8 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
* time relying on p->on_rq.
*/
WARN_ON_ONCE(p->state == TASK_RUNNING &&
- p->sched_class == &fair_sched_class &&
- (p->on_rq && !task_on_rq_migrating(p)));
+ p->sched_class == &fair_sched_class &&
+ (p->on_rq && !task_on_rq_migrating(p)));
#ifdef CONFIG_LOCKDEP
/*
@@ -1371,7 +1383,7 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
* task_rq_lock().
*/
WARN_ON_ONCE(debug_locks && !(lockdep_is_held(&p->pi_lock) ||
- lockdep_is_held(&task_rq(p)->lock)));
+ lockdep_is_held(&task_rq(p)->lock)));
#endif
#endif
@@ -1429,7 +1441,7 @@ static int migrate_swap_stop(void *data)
dst_rq = cpu_rq(arg->dst_cpu);
double_raw_lock(&arg->src_task->pi_lock,
- &arg->dst_task->pi_lock);
+ &arg->dst_task->pi_lock);
double_rq_lock(src_rq, dst_rq);
if (task_cpu(arg->dst_task) != arg->dst_cpu)
@@ -1567,7 +1579,7 @@ unsigned long wait_task_inactive(struct task_struct *p, long match_state)
queued = task_on_rq_queued(p);
ncsw = 0;
if (!match_state || p->state == match_state ||
- p->saved_state == match_state)
+ p->saved_state == match_state)
ncsw = p->nvcsw | LONG_MIN; /* sets MSB */
task_rq_unlock(rq, p, &rf);
@@ -1598,7 +1610,7 @@ unsigned long wait_task_inactive(struct task_struct *p, long match_state)
* yield - it could be a while.
*/
if (unlikely(queued)) {
- ktime_t to = ktime_set(0, NSEC_PER_SEC/HZ);
+ ktime_t to = ktime_set(0, NSEC_PER_SEC / HZ);
set_current_state(TASK_UNINTERRUPTIBLE);
schedule_hrtimeout(&to, HRTIMER_MODE_REL);
@@ -1667,7 +1679,10 @@ static int select_fallback_rq(int cpu, struct task_struct *p)
{
int nid = cpu_to_node(cpu);
const struct cpumask *nodemask = NULL;
- enum { cpuset, possible, fail } state = cpuset;
+
+ enum {
+ cpuset, possible, fail
+ } state = cpuset;
int dest_cpu;
/*
@@ -1688,6 +1703,7 @@ static int select_fallback_rq(int cpu, struct task_struct *p)
}
for (;;) {
+
/* Any allowed, online CPU? */
for_each_cpu(dest_cpu, tsk_cpus_allowed(p)) {
if (!(p->flags & PF_KTHREAD) && !cpu_active(dest_cpu))
@@ -1726,7 +1742,7 @@ static int select_fallback_rq(int cpu, struct task_struct *p)
*/
if (p->mm && printk_ratelimit()) {
printk_deferred("process %d (%s) no longer affine to cpu%d\n",
- task_pid_nr(p), p->comm, cpu);
+ task_pid_nr(p), p->comm, cpu);
}
}
@@ -1757,7 +1773,7 @@ int select_task_rq(struct task_struct *p, int cpu, int sd_flags, int wake_flags)
* not worry about this generic constraint ]
*/
if (unlikely(!cpumask_test_cpu(cpu, tsk_cpus_allowed(p)) ||
- !cpu_online(cpu)))
+ !cpu_online(cpu)))
cpu = select_fallback_rq(task_cpu(p), p);
return cpu;
@@ -1772,7 +1788,7 @@ static void update_avg(u64 *avg, u64 sample)
#else
static inline int __set_cpus_allowed_ptr(struct task_struct *p,
- const struct cpumask *new_mask, bool check)
+ const struct cpumask *new_mask, bool check)
{
return set_cpus_allowed_ptr(p, new_mask);
}
@@ -1798,6 +1814,7 @@ ttwu_stat(struct task_struct *p, int cpu, int wake_flags)
schedstat_inc(p->se.statistics.nr_wakeups_remote);
rcu_read_lock();
+
for_each_domain(rq->cpu, sd) {
if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
schedstat_inc(sd->ttwu_wake_remote);
@@ -1828,7 +1845,7 @@ static inline void ttwu_activate(struct rq *rq, struct task_struct *p, int en_fl
* Mark the task runnable and perform wakeup-preemption.
*/
static void ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags,
- struct pin_cookie cookie)
+ struct pin_cookie cookie)
{
check_preempt_curr(rq, p, wake_flags);
p->state = TASK_RUNNING;
@@ -1847,7 +1864,7 @@ static void ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags,
if (rq->idle_stamp) {
u64 delta = rq_clock(rq) - rq->idle_stamp;
- u64 max = 2*rq->max_idle_balance_cost;
+ u64 max = 2 * rq->max_idle_balance_cost;
update_avg(&rq->avg_idle, delta);
@@ -1861,7 +1878,7 @@ static void ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags,
static void
ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags,
- struct pin_cookie cookie)
+ struct pin_cookie cookie)
{
int en_flags = ENQUEUE_WAKEUP;
@@ -1904,6 +1921,7 @@ static int ttwu_remote(struct task_struct *p, int wake_flags)
}
#ifdef CONFIG_SMP
+
void sched_ttwu_pending(void)
{
struct rq *rq = this_rq();
@@ -2318,18 +2336,18 @@ void __dl_clear_params(struct task_struct *p)
*/
static void __sched_fork(unsigned long clone_flags, struct task_struct *p)
{
- p->on_rq = 0;
+ p->on_rq = 0;
- p->se.on_rq = 0;
- p->se.exec_start = 0;
- p->se.sum_exec_runtime = 0;
- p->se.prev_sum_exec_runtime = 0;
- p->se.nr_migrations = 0;
- p->se.vruntime = 0;
+ p->se.on_rq = 0;
+ p->se.exec_start = 0;
+ p->se.sum_exec_runtime = 0;
+ p->se.prev_sum_exec_runtime = 0;
+ p->se.nr_migrations = 0;
+ p->se.vruntime = 0;
INIT_LIST_HEAD(&p->se.group_node);
#ifdef CONFIG_FAIR_GROUP_SCHED
- p->se.cfs_rq = NULL;
+ p->se.cfs_rq = NULL;
#endif
#ifdef CONFIG_SCHEDSTATS
@@ -2342,10 +2360,10 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p)
__dl_clear_params(p);
INIT_LIST_HEAD(&p->rt.run_list);
- p->rt.timeout = 0;
- p->rt.time_slice = sched_rr_timeslice;
- p->rt.on_rq = 0;
- p->rt.on_list = 0;
+ p->rt.timeout = 0;
+ p->rt.time_slice = sched_rr_timeslice;
+ p->rt.on_rq = 0;
+ p->rt.on_list = 0;
#ifdef CONFIG_PREEMPT_NOTIFIERS
INIT_HLIST_HEAD(&p->preempt_notifiers);
@@ -2387,8 +2405,9 @@ void set_numabalancing_state(bool enabled)
}
#ifdef CONFIG_PROC_SYSCTL
+
int sysctl_numa_balancing(struct ctl_table *table, int write,
- void __user *buffer, size_t *lenp, loff_t *ppos)
+ void __user *buffer, size_t *lenp, loff_t *ppos)
{
struct ctl_table t;
int err;
@@ -2462,8 +2481,9 @@ static void __init init_schedstats(void)
}
#ifdef CONFIG_PROC_SYSCTL
+
int sysctl_schedstats(struct ctl_table *table, int write,
- void __user *buffer, size_t *lenp, loff_t *ppos)
+ void __user *buffer, size_t *lenp, loff_t *ppos)
{
struct ctl_table t;
int err;
@@ -2483,7 +2503,10 @@ int sysctl_schedstats(struct ctl_table *table, int write,
}
#endif /* CONFIG_PROC_SYSCTL */
#else /* !CONFIG_SCHEDSTATS */
-static inline void init_schedstats(void) {}
+
+static inline void init_schedstats(void)
+{
+}
#endif /* CONFIG_SCHEDSTATS */
/*
@@ -2593,10 +2616,11 @@ unsigned long to_ratio(u64 period, u64 runtime)
}
#ifdef CONFIG_SMP
+
inline struct dl_bw *dl_bw_of(int i)
{
RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held(),
- "sched RCU must be held");
+ "sched RCU must be held");
return &cpu_rq(i)->rd->dl_bw;
}
@@ -2606,13 +2630,14 @@ static inline int dl_bw_cpus(int i)
int cpus = 0;
RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held(),
- "sched RCU must be held");
+ "sched RCU must be held");
for_each_cpu_and(i, rd->span, cpu_active_mask)
- cpus++;
+ cpus++;
return cpus;
}
#else
+
inline struct dl_bw *dl_bw_of(int i)
{
return &cpu_rq(i)->dl.dl_bw;
@@ -2636,11 +2661,11 @@ static inline int dl_bw_cpus(int i)
* __setparam_dl().
*/
static int dl_overflow(struct task_struct *p, int policy,
- const struct sched_attr *attr)
+ const struct sched_attr *attr)
{
struct dl_bw *dl_b = dl_bw_of(task_cpu(p));
- u64 period = attr->sched_period ?: attr->sched_deadline;
+ u64 period = attr->sched_period ? : attr->sched_deadline;
u64 runtime = attr->sched_runtime;
u64 new_bw = dl_policy(policy) ? to_ratio(period, runtime) : 0;
int cpus, err = -1;
@@ -2657,11 +2682,11 @@ static int dl_overflow(struct task_struct *p, int policy,
raw_spin_lock(&dl_b->lock);
cpus = dl_bw_cpus(task_cpu(p));
if (dl_policy(policy) && !task_has_dl_policy(p) &&
- !__dl_overflow(dl_b, cpus, 0, new_bw)) {
+ !__dl_overflow(dl_b, cpus, 0, new_bw)) {
__dl_add(dl_b, new_bw);
err = 0;
} else if (dl_policy(policy) && task_has_dl_policy(p) &&
- !__dl_overflow(dl_b, cpus, p->dl.dl_bw, new_bw)) {
+ !__dl_overflow(dl_b, cpus, p->dl.dl_bw, new_bw)) {
__dl_clear(dl_b, p->dl.dl_bw);
__dl_add(dl_b, new_bw);
err = 0;
@@ -2768,7 +2793,7 @@ static void __fire_sched_in_preempt_notifiers(struct task_struct *curr)
struct preempt_notifier *notifier;
hlist_for_each_entry(notifier, &curr->preempt_notifiers, link)
- notifier->ops->sched_in(notifier, raw_smp_processor_id());
+ notifier->ops->sched_in(notifier, raw_smp_processor_id());
}
static __always_inline void fire_sched_in_preempt_notifiers(struct task_struct *curr)
@@ -2779,17 +2804,17 @@ static __always_inline void fire_sched_in_preempt_notifiers(struct task_struct *
static void
__fire_sched_out_preempt_notifiers(struct task_struct *curr,
- struct task_struct *next)
+ struct task_struct *next)
{
struct preempt_notifier *notifier;
hlist_for_each_entry(notifier, &curr->preempt_notifiers, link)
- notifier->ops->sched_out(notifier, next);
+ notifier->ops->sched_out(notifier, next);
}
static __always_inline void
fire_sched_out_preempt_notifiers(struct task_struct *curr,
- struct task_struct *next)
+ struct task_struct *next)
{
if (static_key_false(&preempt_notifier_key))
__fire_sched_out_preempt_notifiers(curr, next);
@@ -2803,7 +2828,7 @@ static inline void fire_sched_in_preempt_notifiers(struct task_struct *curr)
static inline void
fire_sched_out_preempt_notifiers(struct task_struct *curr,
- struct task_struct *next)
+ struct task_struct *next)
{
}
@@ -2824,7 +2849,7 @@ fire_sched_out_preempt_notifiers(struct task_struct *curr,
*/
static inline void
prepare_task_switch(struct rq *rq, struct task_struct *prev,
- struct task_struct *next)
+ struct task_struct *next)
{
sched_info_switch(rq, prev, next);
perf_event_task_sched_out(prev, next);
@@ -2853,7 +2878,7 @@ prepare_task_switch(struct rq *rq, struct task_struct *prev,
* because prev may have moved to another CPU.
*/
static struct rq *finish_task_switch(struct task_struct *prev)
- __releases(rq->lock)
+__releases(rq->lock)
{
struct rq *rq = this_rq();
struct mm_struct *mm = rq->prev_mm;
@@ -2870,9 +2895,9 @@ static struct rq *finish_task_switch(struct task_struct *prev)
*
* Also, see FORK_PREEMPT_COUNT.
*/
- if (WARN_ONCE(preempt_count() != 2*PREEMPT_DISABLE_OFFSET,
- "corrupted preempt_count: %s/%d/0x%x\n",
- current->comm, current->pid, preempt_count()))
+ if (WARN_ONCE(preempt_count() != 2 * PREEMPT_DISABLE_OFFSET,
+ "corrupted preempt_count: %s/%d/0x%x\n",
+ current->comm, current->pid, preempt_count()))
preempt_count_set(FORK_PREEMPT_COUNT);
rq->prev_mm = NULL;
@@ -2954,7 +2979,7 @@ static inline void balance_callback(struct rq *rq)
* @prev: the thread we just switched away from.
*/
asmlinkage __visible void schedule_tail(struct task_struct *prev)
- __releases(rq->lock)
+__releases(rq->lock)
{
struct rq *rq;
@@ -2980,7 +3005,7 @@ asmlinkage __visible void schedule_tail(struct task_struct *prev)
*/
static __always_inline struct rq *
context_switch(struct rq *rq, struct task_struct *prev,
- struct task_struct *next, struct pin_cookie cookie)
+ struct task_struct *next, struct pin_cookie cookie)
{
struct mm_struct *mm, *oldmm;
@@ -3033,7 +3058,7 @@ unsigned long nr_running(void)
unsigned long i, sum = 0;
for_each_online_cpu(i)
- sum += cpu_rq(i)->nr_running;
+ sum += cpu_rq(i)->nr_running;
return sum;
}
@@ -3063,7 +3088,7 @@ unsigned long long nr_context_switches(void)
unsigned long long sum = 0;
for_each_possible_cpu(i)
- sum += cpu_rq(i)->nr_switches;
+ sum += cpu_rq(i)->nr_switches;
return sum;
}
@@ -3073,7 +3098,7 @@ unsigned long nr_iowait(void)
unsigned long i, sum = 0;
for_each_possible_cpu(i)
- sum += atomic_read(&cpu_rq(i)->nr_iowait);
+ sum += atomic_read(&cpu_rq(i)->nr_iowait);
return sum;
}
@@ -3109,7 +3134,7 @@ void sched_exec(void)
goto unlock;
if (likely(cpu_active(dest_cpu))) {
- struct migration_arg arg = { p, dest_cpu };
+ struct migration_arg arg = {p, dest_cpu};
raw_spin_unlock_irqrestore(&p->pi_lock, flags);
stop_one_cpu(task_cpu(p), migration_cpu_stop, &arg);
@@ -3217,6 +3242,7 @@ void scheduler_tick(void)
}
#ifdef CONFIG_NO_HZ_FULL
+
/**
* scheduler_tick_max_deferment
*
@@ -3246,6 +3272,7 @@ u64 scheduler_tick_max_deferment(void)
#if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \
defined(CONFIG_PREEMPT_TRACER))
+
/*
* If the value passed in is equal to the current preempt count
* then we just disabled preemption. Start timing the latency.
@@ -3276,7 +3303,7 @@ void preempt_count_add(int val)
* Spinlock count overflowing soon?
*/
DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >=
- PREEMPT_MASK - 10);
+ PREEMPT_MASK - 10);
#endif
preempt_latency_start(val);
}
@@ -3305,7 +3332,7 @@ void preempt_count_sub(int val)
* Is the spinlock portion underflowing?
*/
if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) &&
- !(preempt_count() & PREEMPT_MASK)))
+ !(preempt_count() & PREEMPT_MASK)))
return;
#endif
@@ -3316,8 +3343,14 @@ EXPORT_SYMBOL(preempt_count_sub);
NOKPROBE_SYMBOL(preempt_count_sub);
#else
-static inline void preempt_latency_start(int val) { }
-static inline void preempt_latency_stop(int val) { }
+
+static inline void preempt_latency_start(int val)
+{
+}
+
+static inline void preempt_latency_stop(int val)
+{
+}
#endif
/*
@@ -3339,7 +3372,7 @@ static noinline void __schedule_bug(struct task_struct *prev)
if (irqs_disabled())
print_irqtrace_events(prev);
if (IS_ENABLED(CONFIG_DEBUG_PREEMPT)
- && in_atomic_preempt_off()) {
+ && in_atomic_preempt_off()) {
pr_err("Preemption disabled at:");
print_ip_sym(preempt_disable_ip);
pr_cont("\n");
@@ -3373,6 +3406,40 @@ static inline void schedule_debug(struct task_struct *prev)
}
#if defined(CONFIG_PREEMPT_RT_FULL) && defined(CONFIG_SMP)
+#define MIGRATE_DISABLE_SET_AFFIN (1<<30) /* Can't make a negative */
+#define migrate_disabled_updated(p) ((p)->migrate_disable & MIGRATE_DISABLE_SET_AFFIN)
+#define migrate_disable_count(p) ((p)->migrate_disable & ~MIGRATE_DISABLE_SET_AFFIN)
+
+static inline void update_migrate_disable(struct task_struct *p)
+{
+ const struct cpumask *mask;
+
+ if (likely(!p->migrate_disable))
+ return;
+
+ /* Did we already update affinity? */
+ if (unlikely(migrate_disabled_updated(p)))
+ return;
+
+ /*
+ * Since this is always current we can get away with only locking
+ * rq->lock, the ->cpus_allowed value can normally only be changed
+ * while holding both p->pi_lock and rq->lock, but seeing that this
+ * is current, we cannot actually be waking up, so all code that
+ * relies on serialization against p->pi_lock is out of scope.
+ *
+ * Having rq->lock serializes us against things like
+ * set_cpus_allowed_ptr() that can still happen concurrently.
+ */
+ mask = tsk_cpus_allowed(p);
+
+ if (p->sched_class->set_cpus_allowed)
+ p->sched_class->set_cpus_allowed(p, mask);
+ p->nr_cpus_allowed = cpumask_weight(mask);
+
+ /* Let migrate_enable know to fix things back up */
+ p->migrate_disable |= MIGRATE_DISABLE_SET_AFFIN;
+}
void migrate_disable(void)
{
@@ -3408,6 +3475,9 @@ EXPORT_SYMBOL(migrate_disable);
void migrate_enable(void)
{
struct task_struct *p = current;
+ const struct cpumask *mask;
+ unsigned long flags;
+ struct rq *rq;
if (in_atomic() || irqs_disabled()) {
#ifdef CONFIG_SCHED_DEBUG
@@ -3424,23 +3494,45 @@ void migrate_enable(void)
#endif
WARN_ON_ONCE(p->migrate_disable <= 0);
- if (p->migrate_disable > 1) {
+ preempt_disable();
+ if (migrate_disable_count(p) > 1) {
p->migrate_disable--;
+ preempt_enable();
return;
}
- preempt_disable();
- /*
- * Clearing migrate_disable causes tsk_cpus_allowed to
- * show the tasks original cpu affinity.
- */
- p->migrate_disable = 0;
+ if (unlikely(migrate_disabled_updated(p))) {
+ /*
+ * Undo whatever update_migrate_disable() did, also see there
+ * about locking.
+ */
+ rq = this_rq();
+ raw_spin_lock_irqsave(&rq->lock, flags);
+
+ /*
+ * Clearing migrate_disable causes tsk_cpus_allowed to
+ * show the tasks original cpu affinity.
+ */
+ p->migrate_disable = 0;
+ mask = tsk_cpus_allowed(p);
+ if (p->sched_class->set_cpus_allowed)
+ p->sched_class->set_cpus_allowed(p, mask);
+ p->nr_cpus_allowed = cpumask_weight(mask);
+ raw_spin_unlock_irqrestore(&rq->lock, flags);
+ } else
+ p->migrate_disable = 0;
unpin_current_cpu();
preempt_enable();
preempt_lazy_enable();
}
EXPORT_SYMBOL(migrate_enable);
+#else
+
+static inline void update_migrate_disable(struct task_struct *p)
+{
+}
+#define migrate_disabled_updated(p) 0
#endif
/*
@@ -3457,7 +3549,7 @@ pick_next_task(struct rq *rq, struct task_struct *prev, struct pin_cookie cookie
* the fair class we can call that function directly:
*/
if (likely(prev->sched_class == class &&
- rq->nr_running == rq->cfs.h_nr_running)) {
+ rq->nr_running == rq->cfs.h_nr_running)) {
p = fair_sched_class.pick_next_task(rq, prev, cookie);
if (unlikely(p == RETRY_TASK))
goto again;
@@ -3552,6 +3644,7 @@ static void __sched notrace __schedule(bool preempt)
rq->clock_skip_update <<= 1; /* promote REQ to ACT */
+ update_migrate_disable(prev);
switch_count = &prev->nivcsw;
if (!preempt && prev->state) {
if (unlikely(signal_pending_state(prev->state, prev))) {
@@ -3606,12 +3699,12 @@ void __noreturn do_task_dead(void)
/* causes final put_task_struct in finish_task_switch(). */
__set_current_state(TASK_DEAD);
- current->flags |= PF_NOFREEZE; /* tell freezer to ignore us */
+ current->flags |= PF_NOFREEZE; /* tell freezer to ignore us */
__schedule(false);
BUG();
/* Avoid "noreturn function does return". */
for (;;)
- cpu_relax(); /* For when BUG is null */
+ cpu_relax(); /* For when BUG is null */
}
static inline void sched_submit_work(struct task_struct *tsk)
@@ -3658,6 +3751,7 @@ asmlinkage __visible void __sched schedule(void)
EXPORT_SYMBOL(schedule);
#ifdef CONFIG_CONTEXT_TRACKING
+
asmlinkage __visible void __sched schedule_user(void)
{
/*
@@ -3718,6 +3812,7 @@ static void __sched notrace preempt_schedule_common(void)
}
#ifdef CONFIG_PREEMPT_LAZY
+
/*
* If TIF_NEED_RESCHED is then we allow to be scheduled away since this is
* set by a RT task. Oterwise we try to avoid beeing scheduled out as long as
@@ -3742,6 +3837,7 @@ static inline int preemptible_lazy(void)
#endif
#ifdef CONFIG_PREEMPT
+
/*
* this is the entry point to schedule() from in-kernel preemption
* off of preempt_enable. Kernel preemptions off return from interrupt
@@ -3855,7 +3951,7 @@ asmlinkage __visible void __sched preempt_schedule_irq(void)
}
int default_wake_function(wait_queue_t *curr, unsigned mode, int wake_flags,
- void *key)
+ void *key)
{
return try_to_wake_up(curr->private, mode, wake_flags);
}
@@ -3967,7 +4063,7 @@ void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task)
*/
if (dl_prio(prio)) {
if (!dl_prio(p->normal_prio) ||
- (pi_task && dl_entity_preempt(&pi_task->dl, &p->dl))) {
+ (pi_task && dl_entity_preempt(&pi_task->dl, &p->dl))) {
p->dl.dl_boosted = 1;
queue_flag |= ENQUEUE_REPLENISH;
} else
@@ -4003,6 +4099,7 @@ void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task)
preempt_enable();
}
#else
+
static inline int rt_effective_prio(struct task_struct *p, int prio)
{
return prio;
@@ -4072,7 +4169,7 @@ int can_nice(const struct task_struct *p, const int nice)
/* convert nice value [19,-20] to rlimit style value [1,40] */
int nice_rlim = nice_to_rlimit(nice);
- return (nice_rlim <= task_rlimit(p, RLIMIT_NICE) ||
+ return(nice_rlim <= task_rlimit(p, RLIMIT_NICE) ||
capable(CAP_SYS_NICE));
}
@@ -4185,7 +4282,7 @@ __setparam_dl(struct task_struct *p, const struct sched_attr *attr)
dl_se->dl_runtime = attr->sched_runtime;
dl_se->dl_deadline = attr->sched_deadline;
- dl_se->dl_period = attr->sched_period ?: dl_se->dl_deadline;
+ dl_se->dl_period = attr->sched_period ? : dl_se->dl_deadline;
dl_se->flags = attr->sched_flags;
dl_se->dl_bw = to_ratio(dl_se->dl_period, dl_se->dl_runtime);
@@ -4214,10 +4311,10 @@ __setparam_dl(struct task_struct *p, const struct sched_attr *attr)
* sched_setparam() passes in -1 for its policy, to let the functions
* it calls know not to change it.
*/
-#define SETPARAM_POLICY -1
+#define SETPARAM_POLICY -1
static void __setscheduler_params(struct task_struct *p,
- const struct sched_attr *attr)
+ const struct sched_attr *attr)
{
int policy = attr->sched_policy;
@@ -4243,7 +4340,7 @@ static void __setscheduler_params(struct task_struct *p,
/* Actually do priority change: must hold pi & rq lock. */
static void __setscheduler(struct rq *rq, struct task_struct *p,
- const struct sched_attr *attr, bool keep_boost)
+ const struct sched_attr *attr, bool keep_boost)
{
__setscheduler_params(p, attr);
@@ -4304,13 +4401,13 @@ __checkparam_dl(const struct sched_attr *attr)
* sure it's not set (mind that period can be equal to zero).
*/
if (attr->sched_deadline & (1ULL << 63) ||
- attr->sched_period & (1ULL << 63))
+ attr->sched_period & (1ULL << 63))
return false;
/* runtime <= deadline <= period (if period != 0) */
if ((attr->sched_period != 0 &&
- attr->sched_period < attr->sched_deadline) ||
- attr->sched_deadline < attr->sched_runtime)
+ attr->sched_period < attr->sched_deadline) ||
+ attr->sched_deadline < attr->sched_runtime)
return false;
return true;
@@ -4327,13 +4424,13 @@ static bool check_same_owner(struct task_struct *p)
rcu_read_lock();
pcred = __task_cred(p);
match = (uid_eq(cred->euid, pcred->euid) ||
- uid_eq(cred->euid, pcred->uid));
+ uid_eq(cred->euid, pcred->uid));
rcu_read_unlock();
return match;
}
static bool dl_param_changed(struct task_struct *p,
- const struct sched_attr *attr)
+ const struct sched_attr *attr)
{
struct sched_dl_entity *dl_se = &p->dl;
@@ -4347,11 +4444,11 @@ static bool dl_param_changed(struct task_struct *p,
}
static int __sched_setscheduler(struct task_struct *p,
- const struct sched_attr *attr,
- bool user, bool pi)
+ const struct sched_attr *attr,
+ bool user, bool pi)
{
int newprio = dl_policy(attr->sched_policy) ? MAX_DL_PRIO - 1 :
- MAX_RT_PRIO - 1 - attr->sched_priority;
+ MAX_RT_PRIO - 1 - attr->sched_priority;
int retval, oldprio, oldpolicy = -1, queued, running;
int new_effective_prio, policy = attr->sched_policy;
const struct sched_class *prev_class;
@@ -4382,11 +4479,11 @@ static int __sched_setscheduler(struct task_struct *p,
* 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL,
* SCHED_BATCH and SCHED_IDLE is 0.
*/
- if ((p->mm && attr->sched_priority > MAX_USER_RT_PRIO-1) ||
- (!p->mm && attr->sched_priority > MAX_RT_PRIO-1))
+ if ((p->mm && attr->sched_priority > MAX_USER_RT_PRIO - 1) ||
+ (!p->mm && attr->sched_priority > MAX_RT_PRIO - 1))
return -EINVAL;
if ((dl_policy(policy) && !__checkparam_dl(attr)) ||
- (rt_policy(policy) != (attr->sched_priority != 0)))
+ (rt_policy(policy) != (attr->sched_priority != 0)))
return -EINVAL;
/*
@@ -4395,13 +4492,13 @@ static int __sched_setscheduler(struct task_struct *p,
if (user && !capable(CAP_SYS_NICE)) {
if (fair_policy(policy)) {
if (attr->sched_nice < task_nice(p) &&
- !can_nice(p, attr->sched_nice))
+ !can_nice(p, attr->sched_nice))
return -EPERM;
}
if (rt_policy(policy)) {
unsigned long rlim_rtprio =
- task_rlimit(p, RLIMIT_RTPRIO);
+ task_rlimit(p, RLIMIT_RTPRIO);
/* can't set/change the rt policy */
if (policy != p->policy && !rlim_rtprio)
@@ -4409,16 +4506,16 @@ static int __sched_setscheduler(struct task_struct *p,
/* can't increase priority */
if (attr->sched_priority > p->rt_priority &&
- attr->sched_priority > rlim_rtprio)
+ attr->sched_priority > rlim_rtprio)
return -EPERM;
}
- /*
- * Can't set/change SCHED_DEADLINE policy at all for now
- * (safest behavior); in the future we would like to allow
- * unprivileged DL tasks to increase their relative deadline
- * or reduce their runtime (both ways reducing utilization)
- */
+ /*
+ * Can't set/change SCHED_DEADLINE policy at all for now
+ * (safest behavior); in the future we would like to allow
+ * unprivileged DL tasks to increase their relative deadline
+ * or reduce their runtime (both ways reducing utilization)
+ */
if (dl_policy(policy))
return -EPERM;
@@ -4488,8 +4585,8 @@ static int __sched_setscheduler(struct task_struct *p,
* assigned.
*/
if (rt_bandwidth_enabled() && rt_policy(policy) &&
- task_group(p)->rt_bandwidth.rt_runtime == 0 &&
- !task_group_is_autogroup(task_group(p))) {
+ task_group(p)->rt_bandwidth.rt_runtime == 0 &&
+ !task_group_is_autogroup(task_group(p))) {
task_rq_unlock(rq, p, &rf);
return -EPERM;
}
@@ -4504,7 +4601,7 @@ static int __sched_setscheduler(struct task_struct *p,
* will also fail if there's no bandwidth available.
*/
if (!cpumask_subset(span, &p->cpus_allowed) ||
- rq->rd->dl_bw.bw == 0) {
+ rq->rd->dl_bw.bw == 0) {
task_rq_unlock(rq, p, &rf);
return -EPERM;
}
@@ -4585,12 +4682,12 @@ static int __sched_setscheduler(struct task_struct *p,
}
static int _sched_setscheduler(struct task_struct *p, int policy,
- const struct sched_param *param, bool check)
+ const struct sched_param *param, bool check)
{
struct sched_attr attr = {
- .sched_policy = policy,
+ .sched_policy = policy,
.sched_priority = param->sched_priority,
- .sched_nice = PRIO_TO_NICE(p->static_prio),
+ .sched_nice = PRIO_TO_NICE(p->static_prio),
};
/* Fixup the legacy SCHED_RESET_ON_FORK hack. */
@@ -4602,6 +4699,7 @@ static int _sched_setscheduler(struct task_struct *p, int policy,
return __sched_setscheduler(p, &attr, check, true);
}
+
/**
* sched_setscheduler - change the scheduling policy and/or RT priority of a thread.
* @p: the task in question.
@@ -4613,7 +4711,7 @@ static int _sched_setscheduler(struct task_struct *p, int policy,
* NOTE that the task may be already dead.
*/
int sched_setscheduler(struct task_struct *p, int policy,
- const struct sched_param *param)
+ const struct sched_param *param)
{
return _sched_setscheduler(p, policy, param, true);
}
@@ -4639,7 +4737,7 @@ EXPORT_SYMBOL_GPL(sched_setattr);
* Return: 0 on success. An error code otherwise.
*/
int sched_setscheduler_nocheck(struct task_struct *p, int policy,
- const struct sched_param *param)
+ const struct sched_param *param)
{
return _sched_setscheduler(p, policy, param, false);
}
@@ -4671,7 +4769,7 @@ do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param)
* Mimics kernel/events/core.c perf_copy_attr().
*/
static int sched_copy_attr(struct sched_attr __user *uattr,
- struct sched_attr *attr)
+ struct sched_attr *attr)
{
u32 size;
int ret;
@@ -4688,10 +4786,10 @@ static int sched_copy_attr(struct sched_attr __user *uattr,
if (ret)
return ret;
- if (size > PAGE_SIZE) /* silly large */
+ if (size > PAGE_SIZE) /* silly large */
goto err_size;
- if (!size) /* abi compat */
+ if (!size) /* abi compat */
size = SCHED_ATTR_SIZE_VER0;
if (size < SCHED_ATTR_SIZE_VER0)
@@ -4708,8 +4806,8 @@ static int sched_copy_attr(struct sched_attr __user *uattr,
unsigned char __user *end;
unsigned char val;
- addr = (void __user *)uattr + sizeof(*attr);
- end = (void __user *)uattr + size;
+ addr = (void __user *) uattr + sizeof(*attr);
+ end = (void __user *) uattr + size;
for (; addr < end; addr++) {
ret = get_user(val, addr);
@@ -4747,7 +4845,7 @@ static int sched_copy_attr(struct sched_attr __user *uattr,
* Return: 0 on success. An error code otherwise.
*/
SYSCALL_DEFINE3(sched_setscheduler, pid_t, pid, int, policy,
- struct sched_param __user *, param)
+ struct sched_param __user *, param)
{
/* negative values for policy are not valid */
if (policy < 0)
@@ -4775,7 +4873,7 @@ SYSCALL_DEFINE2(sched_setparam, pid_t, pid, struct sched_param __user *, param)
* @flags: for future extension.
*/
SYSCALL_DEFINE3(sched_setattr, pid_t, pid, struct sched_attr __user *, uattr,
- unsigned int, flags)
+ unsigned int, flags)
{
struct sched_attr attr;
struct task_struct *p;
@@ -4788,7 +4886,7 @@ SYSCALL_DEFINE3(sched_setattr, pid_t, pid, struct sched_attr __user *, uattr,
if (retval)
return retval;
- if ((int)attr.sched_policy < 0)
+ if ((int) attr.sched_policy < 0)
return -EINVAL;
rcu_read_lock();
@@ -4823,7 +4921,7 @@ SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid)
retval = security_task_getscheduler(p);
if (!retval)
retval = p->policy
- | (p->sched_reset_on_fork ? SCHED_RESET_ON_FORK : 0);
+ | (p->sched_reset_on_fork ? SCHED_RESET_ON_FORK : 0);
}
rcu_read_unlock();
return retval;
@@ -4839,7 +4937,7 @@ SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid)
*/
SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param)
{
- struct sched_param lp = { .sched_priority = 0 };
+ struct sched_param lp = {.sched_priority = 0};
struct task_struct *p;
int retval;
@@ -4873,8 +4971,8 @@ SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param)
}
static int sched_read_attr(struct sched_attr __user *uattr,
- struct sched_attr *attr,
- unsigned int usize)
+ struct sched_attr *attr,
+ unsigned int usize)
{
int ret;
@@ -4890,8 +4988,8 @@ static int sched_read_attr(struct sched_attr __user *uattr,
unsigned char *addr;
unsigned char *end;
- addr = (void *)attr + usize;
- end = (void *)attr + sizeof(*attr);
+ addr = (void *) attr + usize;
+ end = (void *) attr + sizeof(*attr);
for (; addr < end; addr++) {
if (*addr)
@@ -4916,7 +5014,7 @@ static int sched_read_attr(struct sched_attr __user *uattr,
* @flags: for future extension.
*/
SYSCALL_DEFINE4(sched_getattr, pid_t, pid, struct sched_attr __user *, uattr,
- unsigned int, size, unsigned int, flags)
+ unsigned int, size, unsigned int, flags)
{
struct sched_attr attr = {
.size = sizeof(struct sched_attr),
@@ -4925,7 +5023,7 @@ SYSCALL_DEFINE4(sched_getattr, pid_t, pid, struct sched_attr __user *, uattr,
int retval;
if (!uattr || pid < 0 || size > PAGE_SIZE ||
- size < SCHED_ATTR_SIZE_VER0 || flags)
+ size < SCHED_ATTR_SIZE_VER0 || flags)
return -EINVAL;
rcu_read_lock();
@@ -5048,7 +5146,7 @@ long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
}
static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
- struct cpumask *new_mask)
+ struct cpumask *new_mask)
{
if (len < cpumask_size())
cpumask_clear(new_mask);
@@ -5067,7 +5165,7 @@ static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
* Return: 0 on success. An error code otherwise.
*/
SYSCALL_DEFINE3(sched_setaffinity, pid_t, pid, unsigned int, len,
- unsigned long __user *, user_mask_ptr)
+ unsigned long __user *, user_mask_ptr)
{
cpumask_var_t new_mask;
int retval;
@@ -5119,14 +5217,14 @@ long sched_getaffinity(pid_t pid, struct cpumask *mask)
* error code otherwise.
*/
SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len,
- unsigned long __user *, user_mask_ptr)
+ unsigned long __user *, user_mask_ptr)
{
int ret;
cpumask_var_t mask;
if ((len * BITS_PER_BYTE) < nr_cpu_ids)
return -EINVAL;
- if (len & (sizeof(unsigned long)-1))
+ if (len & (sizeof(unsigned long) - 1))
return -EINVAL;
if (!alloc_cpumask_var(&mask, GFP_KERNEL))
@@ -5176,6 +5274,7 @@ SYSCALL_DEFINE0(sched_yield)
}
#ifndef CONFIG_PREEMPT
+
int __sched _cond_resched(void)
{
if (should_resched(0)) {
@@ -5216,6 +5315,7 @@ int __cond_resched_lock(spinlock_t *lock)
EXPORT_SYMBOL(__cond_resched_lock);
#ifndef CONFIG_PREEMPT_RT_FULL
+
int __sched __cond_resched_softirq(void)
{
BUG_ON(!in_softirq());
@@ -5374,7 +5474,7 @@ SYSCALL_DEFINE1(sched_get_priority_max, int, policy)
switch (policy) {
case SCHED_FIFO:
case SCHED_RR:
- ret = MAX_USER_RT_PRIO-1;
+ ret = MAX_USER_RT_PRIO - 1;
break;
case SCHED_DEADLINE:
case SCHED_NORMAL:
@@ -5424,7 +5524,7 @@ SYSCALL_DEFINE1(sched_get_priority_min, int, policy)
* an error code.
*/
SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid,
- struct timespec __user *, interval)
+ struct timespec __user *, interval)
{
struct task_struct *p;
unsigned int time_slice;
@@ -5488,7 +5588,7 @@ void sched_show_task(struct task_struct *p)
rcu_read_unlock();
printk(KERN_CONT "%5lu %5d %6d 0x%08lx\n", free,
task_pid_nr(p), ppid,
- (unsigned long)task_thread_info(p)->flags);
+ (unsigned long) task_thread_info(p)->flags);
print_worker_info(KERN_INFO, p);
show_stack(p, NULL);
@@ -5507,6 +5607,7 @@ void show_state_filter(unsigned long state_filter)
" task PC stack pid father\n");
#endif
rcu_read_lock();
+
for_each_process_thread(g, p) {
/*
* reset the NMI-timeout, listing all files on a slow
@@ -5608,7 +5709,7 @@ void init_idle(struct task_struct *idle, int cpu)
}
int cpuset_cpumask_can_shrink(const struct cpumask *cur,
- const struct cpumask *trial)
+ const struct cpumask *trial)
{
int ret = 1, trial_cpus;
struct dl_bw *cur_dl_b;
@@ -5623,7 +5724,7 @@ int cpuset_cpumask_can_shrink(const struct cpumask *cur,
raw_spin_lock_irqsave(&cur_dl_b->lock, flags);
if (cur_dl_b->bw != -1 &&
- cur_dl_b->bw * trial_cpus < cur_dl_b->total_bw)
+ cur_dl_b->bw * trial_cpus < cur_dl_b->total_bw)
ret = 0;
raw_spin_unlock_irqrestore(&cur_dl_b->lock, flags);
rcu_read_unlock_sched();
@@ -5632,7 +5733,7 @@ int cpuset_cpumask_can_shrink(const struct cpumask *cur,
}
int task_can_attach(struct task_struct *p,
- const struct cpumask *cs_cpus_allowed)
+ const struct cpumask *cs_cpus_allowed)
{
int ret = 0;
@@ -5652,9 +5753,9 @@ int task_can_attach(struct task_struct *p,
#ifdef CONFIG_SMP
if (dl_task(p) && !cpumask_intersects(task_rq(p)->rd->span,
- cs_cpus_allowed)) {
+ cs_cpus_allowed)) {
unsigned int dest_cpu = cpumask_any_and(cpu_active_mask,
- cs_cpus_allowed);
+ cs_cpus_allowed);
struct dl_bw *dl_b;
bool overflow;
int cpus;
@@ -5690,10 +5791,11 @@ int task_can_attach(struct task_struct *p,
static bool sched_smp_initialized __read_mostly;
#ifdef CONFIG_NUMA_BALANCING
+
/* Migrate current task p to target_cpu */
int migrate_task_to(struct task_struct *p, int target_cpu)
{
- struct migration_arg arg = { p, target_cpu };
+ struct migration_arg arg = {p, target_cpu};
int curr_cpu = task_cpu(p);
if (curr_cpu == target_cpu)
@@ -5940,7 +6042,7 @@ static inline bool sched_debug(void)
}
static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
- struct cpumask *groupmask)
+ struct cpumask *groupmask)
{
struct sched_group *group = sd->groups;
@@ -5951,21 +6053,20 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
if (!(sd->flags & SD_LOAD_BALANCE)) {
printk("does not load-balance\n");
if (sd->parent)
- printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain"
- " has parent");
+ printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain has parent");
return -1;
}
printk(KERN_CONT "span %*pbl level %s\n",
- cpumask_pr_args(sched_domain_span(sd)), sd->name);
+ cpumask_pr_args(sched_domain_span(sd)), sd->name);
if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) {
- printk(KERN_ERR "ERROR: domain->span does not contain "
- "CPU%d\n", cpu);
+ printk(KERN_ERR "ERROR: domain->span does not contain CPU%d\n",
+ cpu);
}
if (!cpumask_test_cpu(cpu, sched_group_cpus(group))) {
- printk(KERN_ERR "ERROR: domain->groups does not contain"
- " CPU%d\n", cpu);
+ printk(KERN_ERR "ERROR: domain->groups does not contain CPU%d\n",
+ cpu);
}
printk(KERN_DEBUG "%*s groups:", level + 1, "");
@@ -5983,7 +6084,7 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
}
if (!(sd->flags & SD_OVERLAP) &&
- cpumask_intersects(groupmask, sched_group_cpus(group))) {
+ cpumask_intersects(groupmask, sched_group_cpus(group))) {
printk(KERN_CONT "\n");
printk(KERN_ERR "ERROR: repeated CPUs\n");
break;
@@ -5992,7 +6093,7 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
cpumask_or(groupmask, groupmask, sched_group_cpus(group));
printk(KERN_CONT " %*pbl",
- cpumask_pr_args(sched_group_cpus(group)));
+ cpumask_pr_args(sched_group_cpus(group)));
if (group->sgc->capacity != SCHED_CAPACITY_SCALE) {
printk(KERN_CONT " (cpu_capacity = %d)",
group->sgc->capacity);
@@ -6006,9 +6107,8 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
printk(KERN_ERR "ERROR: groups don't span domain->span\n");
if (sd->parent &&
- !cpumask_subset(groupmask, sched_domain_span(sd->parent)))
- printk(KERN_ERR "ERROR: parent span is not a superset "
- "of domain->span\n");
+ !cpumask_subset(groupmask, sched_domain_span(sd->parent)))
+ printk(KERN_ERR "ERROR: parent span is not a superset of domain->span\n");
return 0;
}
@@ -6037,8 +6137,9 @@ static void sched_domain_debug(struct sched_domain *sd, int cpu)
}
#else /* !CONFIG_SCHED_DEBUG */
-# define sched_debug_enabled 0
-# define sched_domain_debug(sd, cpu) do { } while (0)
+#define sched_debug_enabled 0
+#define sched_domain_debug(sd, cpu) do { } while (0)
+
static inline bool sched_debug(void)
{
return false;
@@ -6052,13 +6153,13 @@ static int sd_degenerate(struct sched_domain *sd)
/* Following flags need at least 2 groups */
if (sd->flags & (SD_LOAD_BALANCE |
- SD_BALANCE_NEWIDLE |
- SD_BALANCE_FORK |
- SD_BALANCE_EXEC |
- SD_SHARE_CPUCAPACITY |
- SD_ASYM_CPUCAPACITY |
- SD_SHARE_PKG_RESOURCES |
- SD_SHARE_POWERDOMAIN)) {
+ SD_BALANCE_NEWIDLE |
+ SD_BALANCE_FORK |
+ SD_BALANCE_EXEC |
+ SD_SHARE_CPUCAPACITY |
+ SD_ASYM_CPUCAPACITY |
+ SD_SHARE_PKG_RESOURCES |
+ SD_SHARE_POWERDOMAIN)) {
if (sd->groups != sd->groups->next)
return 0;
}
@@ -6084,14 +6185,14 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
/* Flags needing groups don't count if only 1 group in parent */
if (parent->groups == parent->groups->next) {
pflags &= ~(SD_LOAD_BALANCE |
- SD_BALANCE_NEWIDLE |
- SD_BALANCE_FORK |
- SD_BALANCE_EXEC |
- SD_ASYM_CPUCAPACITY |
- SD_SHARE_CPUCAPACITY |
- SD_SHARE_PKG_RESOURCES |
- SD_PREFER_SIBLING |
- SD_SHARE_POWERDOMAIN);
+ SD_BALANCE_NEWIDLE |
+ SD_BALANCE_FORK |
+ SD_BALANCE_EXEC |
+ SD_ASYM_CPUCAPACITY |
+ SD_SHARE_CPUCAPACITY |
+ SD_SHARE_PKG_RESOURCES |
+ SD_PREFER_SIBLING |
+ SD_SHARE_POWERDOMAIN);
if (nr_node_ids == 1)
pflags &= ~SD_SERIALIZE;
}
@@ -6319,7 +6420,7 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
struct sched_domain *tmp;
/* Remove the sched domains which do not contribute to scheduling. */
- for (tmp = sd; tmp; ) {
+ for (tmp = sd; tmp;) {
struct sched_domain *parent = tmp->parent;
if (!parent)
break;
@@ -6375,7 +6476,7 @@ __setup("isolcpus=", isolated_cpu_setup);
struct s_data {
struct sched_domain ** __percpu sd;
- struct root_domain *rd;
+ struct root_domain *rd;
};
enum s_alloc {
@@ -6451,7 +6552,7 @@ build_overlap_sched_groups(struct sched_domain *sd, int cpu)
cpumask_clear(covered);
- for_each_cpu_wrap(i, span, cpu) {
+ for_each_cpu_wrap(i, span, cpu) {
struct cpumask *sg_span;
if (cpumask_test_cpu(i, covered))
@@ -6464,7 +6565,7 @@ build_overlap_sched_groups(struct sched_domain *sd, int cpu)
continue;
sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(),
- GFP_KERNEL, cpu_to_node(cpu));
+ GFP_KERNEL, cpu_to_node(cpu));
if (!sg)
goto fail;
@@ -6494,7 +6595,7 @@ build_overlap_sched_groups(struct sched_domain *sd, int cpu)
* breaks. See update_sg_lb_stats().
*/
if ((!groups && cpumask_test_cpu(cpu, sg_span)) ||
- group_balance_cpu(sg) == cpu)
+ group_balance_cpu(sg) == cpu)
groups = sg;
if (!first)
@@ -6632,7 +6733,7 @@ static int __init setup_relax_domain_level(char *str)
__setup("relax_domain_level=", setup_relax_domain_level);
static void set_domain_attribute(struct sched_domain *sd,
- struct sched_domain_attr *attr)
+ struct sched_domain_attr *attr)
{
int request;
@@ -6645,10 +6746,10 @@ static void set_domain_attribute(struct sched_domain *sd,
request = attr->relax_domain_level;
if (request < sd->level) {
/* turn off idle balance on this domain */
- sd->flags &= ~(SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE);
+ sd->flags &= ~(SD_BALANCE_WAKE | SD_BALANCE_NEWIDLE);
} else {
/* turn on idle balance on this domain */
- sd->flags |= (SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE);
+ sd->flags |= (SD_BALANCE_WAKE | SD_BALANCE_NEWIDLE);
}
}
@@ -6656,7 +6757,7 @@ static void __sdt_free(const struct cpumask *cpu_map);
static int __sdt_alloc(const struct cpumask *cpu_map);
static void __free_domain_allocs(struct s_data *d, enum s_alloc what,
- const struct cpumask *cpu_map)
+ const struct cpumask *cpu_map)
{
switch (what) {
case sa_rootdomain:
@@ -6672,7 +6773,7 @@ static void __free_domain_allocs(struct s_data *d, enum s_alloc what,
}
static enum s_alloc __visit_domain_allocation_hell(struct s_data *d,
- const struct cpumask *cpu_map)
+ const struct cpumask *cpu_map)
{
memset(d, 0, sizeof(*d));
@@ -6736,12 +6837,12 @@ static int sched_domains_curr_level;
*
* SD_ASYM_PACKING - describes SMT quirks
*/
-#define TOPOLOGY_SD_FLAGS \
- (SD_SHARE_CPUCAPACITY | \
- SD_SHARE_PKG_RESOURCES | \
- SD_NUMA | \
- SD_ASYM_PACKING | \
- SD_ASYM_CPUCAPACITY | \
+#define TOPOLOGY_SD_FLAGS \
+ (SD_SHARE_CPUCAPACITY | \
+ SD_SHARE_PKG_RESOURCES | \
+ SD_NUMA | \
+ SD_ASYM_PACKING | \
+ SD_ASYM_CPUCAPACITY | \
SD_SHARE_POWERDOMAIN)
static struct sched_domain *
@@ -6765,44 +6866,44 @@ sd_init(struct sched_domain_topology_level *tl,
if (tl->sd_flags)
sd_flags = (*tl->sd_flags)();
if (WARN_ONCE(sd_flags & ~TOPOLOGY_SD_FLAGS,
- "wrong sd_flags in topology description\n"))
+ "wrong sd_flags in topology description\n"))
sd_flags &= ~TOPOLOGY_SD_FLAGS;
*sd = (struct sched_domain){
- .min_interval = sd_weight,
- .max_interval = 2*sd_weight,
- .busy_factor = 32,
- .imbalance_pct = 125,
-
- .cache_nice_tries = 0,
- .busy_idx = 0,
- .idle_idx = 0,
- .newidle_idx = 0,
- .wake_idx = 0,
- .forkexec_idx = 0,
-
- .flags = 1*SD_LOAD_BALANCE
- | 1*SD_BALANCE_NEWIDLE
- | 1*SD_BALANCE_EXEC
- | 1*SD_BALANCE_FORK
- | 0*SD_BALANCE_WAKE
- | 1*SD_WAKE_AFFINE
- | 0*SD_SHARE_CPUCAPACITY
- | 0*SD_SHARE_PKG_RESOURCES
- | 0*SD_SERIALIZE
- | 0*SD_PREFER_SIBLING
- | 0*SD_NUMA
- | sd_flags
- ,
-
- .last_balance = jiffies,
- .balance_interval = sd_weight,
- .smt_gain = 0,
- .max_newidle_lb_cost = 0,
- .next_decay_max_lb_cost = jiffies,
- .child = child,
+ .min_interval = sd_weight,
+ .max_interval = 2 * sd_weight,
+ .busy_factor = 32,
+ .imbalance_pct = 125,
+
+ .cache_nice_tries = 0,
+ .busy_idx = 0,
+ .idle_idx = 0,
+ .newidle_idx = 0,
+ .wake_idx = 0,
+ .forkexec_idx = 0,
+
+ .flags = 1 * SD_LOAD_BALANCE
+ | 1 * SD_BALANCE_NEWIDLE
+ | 1 * SD_BALANCE_EXEC
+ | 1 * SD_BALANCE_FORK
+ | 0 * SD_BALANCE_WAKE
+ | 1 * SD_WAKE_AFFINE
+ | 0 * SD_SHARE_CPUCAPACITY
+ | 0 * SD_SHARE_PKG_RESOURCES
+ | 0 * SD_SERIALIZE
+ | 0 * SD_PREFER_SIBLING
+ | 0 * SD_NUMA
+ | sd_flags
+ ,
+
+ .last_balance = jiffies,
+ .balance_interval = sd_weight,
+ .smt_gain = 0,
+ .max_newidle_lb_cost = 0,
+ .next_decay_max_lb_cost = jiffies,
+ .child = child,
#ifdef CONFIG_SCHED_DEBUG
- .name = tl->name,
+ .name = tl->name,
#endif
};
@@ -6817,7 +6918,7 @@ sd_init(struct sched_domain_topology_level *tl,
struct sched_domain *t = sd;
for_each_lower_domain(t)
- t->flags |= SD_BALANCE_WAKE;
+ t->flags |= SD_BALANCE_WAKE;
}
if (sd->flags & SD_SHARE_CPUCAPACITY) {
@@ -6839,8 +6940,8 @@ sd_init(struct sched_domain_topology_level *tl,
sd->flags |= SD_SERIALIZE;
if (sched_domains_numa_distance[tl->numa_level] > RECLAIM_DISTANCE) {
sd->flags &= ~(SD_BALANCE_EXEC |
- SD_BALANCE_FORK |
- SD_WAKE_AFFINE);
+ SD_BALANCE_FORK |
+ SD_WAKE_AFFINE);
}
#endif
@@ -6871,19 +6972,19 @@ sd_init(struct sched_domain_topology_level *tl,
*/
static struct sched_domain_topology_level default_topology[] = {
#ifdef CONFIG_SCHED_SMT
- { cpu_smt_mask, cpu_smt_flags, SD_INIT_NAME(SMT) },
+ { cpu_smt_mask, cpu_smt_flags, SD_INIT_NAME(SMT)},
#endif
#ifdef CONFIG_SCHED_MC
- { cpu_coregroup_mask, cpu_core_flags, SD_INIT_NAME(MC) },
+ { cpu_coregroup_mask, cpu_core_flags, SD_INIT_NAME(MC)},
#endif
- { cpu_cpu_mask, SD_INIT_NAME(DIE) },
- { NULL, },
+ { cpu_cpu_mask, SD_INIT_NAME(DIE)},
+ { NULL,},
};
static struct sched_domain_topology_level *sched_domain_topology =
default_topology;
-#define for_each_sd_topology(tl) \
+#define for_each_sd_topology(tl) \
for (tl = sched_domain_topology; tl->mask; tl++)
void set_sched_topology(struct sched_domain_topology_level *tl)
@@ -6904,7 +7005,7 @@ static const struct cpumask *sd_numa_mask(int cpu)
static void sched_numa_warn(const char *str)
{
static int done = false;
- int i,j;
+ int i, j;
if (done)
return;
@@ -6916,7 +7017,7 @@ static void sched_numa_warn(const char *str)
for (i = 0; i < nr_node_ids; i++) {
printk(KERN_WARNING " ");
for (j = 0; j < nr_node_ids; j++)
- printk(KERN_CONT "%02d ", node_distance(i,j));
+ printk(KERN_CONT "%02d ", node_distance(i, j));
printk(KERN_CONT "\n");
}
printk(KERN_WARNING "\n");
@@ -6967,7 +7068,8 @@ static void init_numa_topology_type(void)
return;
}
- for_each_online_node(a) {
+ for_each_online_node(a) {
+
for_each_online_node(b) {
/* Find two nodes furthest removed from each other. */
if (node_distance(a, b) < n)
@@ -6976,9 +7078,9 @@ static void init_numa_topology_type(void)
/* Is there an intermediary node between a and b? */
for_each_online_node(c) {
if (node_distance(a, c) < n &&
- node_distance(b, c) < n) {
+ node_distance(b, c) < n) {
sched_numa_topology_type =
- NUMA_GLUELESS_MESH;
+ NUMA_GLUELESS_MESH;
return;
}
}
@@ -7014,8 +7116,8 @@ static void sched_init_numa(void)
int distance = node_distance(i, k);
if (distance > curr_distance &&
- (distance < next_distance ||
- next_distance == curr_distance))
+ (distance < next_distance ||
+ next_distance == curr_distance))
next_distance = distance;
/*
@@ -7099,7 +7201,7 @@ static void sched_init_numa(void)
for (i = 0; sched_domain_topology[i].mask; i++);
tl = kzalloc((i + level + 1) *
- sizeof(struct sched_domain_topology_level), GFP_KERNEL);
+ sizeof(struct sched_domain_topology_level), GFP_KERNEL);
if (!tl)
return;
@@ -7154,9 +7256,18 @@ static void sched_domains_numa_masks_clear(unsigned int cpu)
}
#else
-static inline void sched_init_numa(void) { }
-static void sched_domains_numa_masks_set(unsigned int cpu) { }
-static void sched_domains_numa_masks_clear(unsigned int cpu) { }
+
+static inline void sched_init_numa(void)
+{
+}
+
+static void sched_domains_numa_masks_set(unsigned int cpu)
+{
+}
+
+static void sched_domains_numa_masks_clear(unsigned int cpu)
+{
+}
#endif /* CONFIG_NUMA */
static int __sdt_alloc(const struct cpumask *cpu_map)
@@ -7190,21 +7301,21 @@ static int __sdt_alloc(const struct cpumask *cpu_map)
struct sched_group_capacity *sgc;
sd = kzalloc_node(sizeof(struct sched_domain) + cpumask_size(),
- GFP_KERNEL, cpu_to_node(j));
+ GFP_KERNEL, cpu_to_node(j));
if (!sd)
return -ENOMEM;
*per_cpu_ptr(sdd->sd, j) = sd;
sds = kzalloc_node(sizeof(struct sched_domain_shared),
- GFP_KERNEL, cpu_to_node(j));
+ GFP_KERNEL, cpu_to_node(j));
if (!sds)
return -ENOMEM;
*per_cpu_ptr(sdd->sds, j) = sds;
sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(),
- GFP_KERNEL, cpu_to_node(j));
+ GFP_KERNEL, cpu_to_node(j));
if (!sg)
return -ENOMEM;
@@ -7213,7 +7324,7 @@ static int __sdt_alloc(const struct cpumask *cpu_map)
*per_cpu_ptr(sdd->sg, j) = sg;
sgc = kzalloc_node(sizeof(struct sched_group_capacity) + cpumask_size(),
- GFP_KERNEL, cpu_to_node(j));
+ GFP_KERNEL, cpu_to_node(j));
if (!sgc)
return -ENOMEM;
@@ -7261,8 +7372,8 @@ static void __sdt_free(const struct cpumask *cpu_map)
}
struct sched_domain *build_sched_domain(struct sched_domain_topology_level *tl,
- const struct cpumask *cpu_map, struct sched_domain_attr *attr,
- struct sched_domain *child, int cpu)
+ const struct cpumask *cpu_map, struct sched_domain_attr *attr,
+ struct sched_domain *child, int cpu)
{
struct sched_domain *sd = sd_init(tl, cpu_map, child, cpu);
@@ -7272,16 +7383,16 @@ struct sched_domain *build_sched_domain(struct sched_domain_topology_level *tl,
child->parent = sd;
if (!cpumask_subset(sched_domain_span(child),
- sched_domain_span(sd))) {
+ sched_domain_span(sd))) {
pr_err("BUG: arch topology borken\n");
#ifdef CONFIG_SCHED_DEBUG
pr_err(" the %s domain not a subset of the %s domain\n",
- child->name, sd->name);
+ child->name, sd->name);
#endif
/* Fixup, ensure @sd has at least @child cpus. */
cpumask_or(sched_domain_span(sd),
- sched_domain_span(sd),
- sched_domain_span(child));
+ sched_domain_span(sd),
+ sched_domain_span(child));
}
}
@@ -7295,7 +7406,7 @@ struct sched_domain *build_sched_domain(struct sched_domain_topology_level *tl,
* to the individual cpus
*/
static int build_sched_domains(const struct cpumask *cpu_map,
- struct sched_domain_attr *attr)
+ struct sched_domain_attr *attr)
{
enum s_alloc alloc_state;
struct sched_domain *sd;
@@ -7312,6 +7423,7 @@ static int build_sched_domains(const struct cpumask *cpu_map,
struct sched_domain_topology_level *tl;
sd = NULL;
+
for_each_sd_topology(tl) {
sd = build_sched_domain(tl, cpu_map, attr, sd, i);
if (tl == sched_domain_topology)
@@ -7338,7 +7450,7 @@ static int build_sched_domains(const struct cpumask *cpu_map,
}
/* Calculate CPU capacity for physical packages and nodes */
- for (i = nr_cpumask_bits-1; i >= 0; i--) {
+ for (i = nr_cpumask_bits - 1; i >= 0; i--) {
if (!cpumask_test_cpu(i, cpu_map))
continue;
@@ -7350,6 +7462,7 @@ static int build_sched_domains(const struct cpumask *cpu_map,
/* Attach the domains */
rcu_read_lock();
+
for_each_cpu(i, cpu_map) {
rq = cpu_rq(i);
sd = *per_cpu_ptr(d.sd, i);
@@ -7373,10 +7486,10 @@ static int build_sched_domains(const struct cpumask *cpu_map,
return ret;
}
-static cpumask_var_t *doms_cur; /* current sched domains */
-static int ndoms_cur; /* number of sched domains in 'doms_cur' */
+static cpumask_var_t *doms_cur; /* current sched domains */
+static int ndoms_cur; /* number of sched domains in 'doms_cur' */
static struct sched_domain_attr *dattr_cur;
- /* attribues of custom domains in 'doms_cur' */
+/* attribues of custom domains in 'doms_cur' */
/*
* Special case: If a kmalloc of a doms_cur partition (array of
@@ -7451,13 +7564,13 @@ static void detach_destroy_domains(const struct cpumask *cpu_map)
rcu_read_lock();
for_each_cpu(i, cpu_map)
- cpu_attach_domain(NULL, &def_root_domain, i);
+ cpu_attach_domain(NULL, &def_root_domain, i);
rcu_read_unlock();
}
/* handle null as "default" */
static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
- struct sched_domain_attr *new, int idx_new)
+ struct sched_domain_attr *new, int idx_new)
{
struct sched_domain_attr tmp;
@@ -7467,8 +7580,8 @@ static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
tmp = SD_ATTR_INIT;
return !memcmp(cur ? (cur + idx_cur) : &tmp,
- new ? (new + idx_new) : &tmp,
- sizeof(struct sched_domain_attr));
+ new ? (new + idx_new) : &tmp,
+ sizeof(struct sched_domain_attr));
}
/*
@@ -7498,7 +7611,7 @@ static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
* Call with hotplug lock held
*/
void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
- struct sched_domain_attr *dattr_new)
+ struct sched_domain_attr *dattr_new)
{
int i, j, n;
int new_topology;
@@ -7517,7 +7630,7 @@ void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
for (i = 0; i < ndoms_cur; i++) {
for (j = 0; j < n && !new_topology; j++) {
if (cpumask_equal(doms_cur[i], doms_new[j])
- && dattrs_equal(dattr_cur, i, dattr_new, j))
+ && dattrs_equal(dattr_cur, i, dattr_new, j))
goto match1;
}
/* no match - a current sched domain not in new doms_new[] */
@@ -7538,7 +7651,7 @@ void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
for (i = 0; i < ndoms_new; i++) {
for (j = 0; j < n && !new_topology; j++) {
if (cpumask_equal(doms_new[i], doms_cur[j])
- && dattrs_equal(dattr_new, i, dattr_cur, j))
+ && dattrs_equal(dattr_new, i, dattr_cur, j))
goto match2;
}
/* no match - add a new doms_new */
@@ -7550,7 +7663,7 @@ void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
/* Remember the new sched domains */
if (doms_cur != &fallback_doms)
free_sched_domains(doms_cur, ndoms_cur);
- kfree(dattr_cur); /* kfree(NULL) is safe */
+ kfree(dattr_cur); /* kfree(NULL) is safe */
doms_cur = doms_new;
dattr_cur = dattr_new;
ndoms_cur = ndoms_new;
@@ -7560,7 +7673,7 @@ void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
mutex_unlock(&sched_domains_mutex);
}
-static int num_cpus_frozen; /* used to mark begin/end of suspend/resume */
+static int num_cpus_frozen; /* used to mark begin/end of suspend/resume */
/*
* Update cpusets according to cpu_active mask. If cpusets are
@@ -7701,6 +7814,7 @@ int sched_cpu_starting(unsigned int cpu)
}
#ifdef CONFIG_HOTPLUG_CPU
+
int sched_cpu_dying(unsigned int cpu)
{
struct rq *rq = cpu_rq(cpu);
@@ -7768,6 +7882,7 @@ static int __init migration_init(void)
early_initcall(migration_init);
#else
+
void __init sched_init_smp(void)
{
sched_init_granularity();
@@ -7777,8 +7892,8 @@ void __init sched_init_smp(void)
int in_sched_functions(unsigned long addr)
{
return in_lock_functions(addr) ||
- (addr >= (unsigned long)__sched_text_start
- && addr < (unsigned long)__sched_text_end);
+ (addr >= (unsigned long) __sched_text_start
+ && addr < (unsigned long) __sched_text_end);
}
#ifdef CONFIG_CGROUP_SCHED
@@ -7803,7 +7918,7 @@ static wait_queue_head_t bit_wait_table[WAIT_TABLE_SIZE] __cacheline_aligned;
wait_queue_head_t *bit_waitqueue(void *word, int bit)
{
const int shift = BITS_PER_LONG == 32 ? 5 : 6;
- unsigned long val = (unsigned long)word << shift | bit;
+ unsigned long val = (unsigned long) word << shift | bit;
return bit_wait_table + hash_long(val, WAIT_TABLE_BITS);
}
@@ -7824,38 +7939,39 @@ void __init sched_init(void)
alloc_size += 2 * nr_cpu_ids * sizeof(void **);
#endif
if (alloc_size) {
- ptr = (unsigned long)kzalloc(alloc_size, GFP_NOWAIT);
+ ptr = (unsigned long) kzalloc(alloc_size, GFP_NOWAIT);
#ifdef CONFIG_FAIR_GROUP_SCHED
- root_task_group.se = (struct sched_entity **)ptr;
+ root_task_group.se = (struct sched_entity **) ptr;
ptr += nr_cpu_ids * sizeof(void **);
- root_task_group.cfs_rq = (struct cfs_rq **)ptr;
+ root_task_group.cfs_rq = (struct cfs_rq **) ptr;
ptr += nr_cpu_ids * sizeof(void **);
#endif /* CONFIG_FAIR_GROUP_SCHED */
#ifdef CONFIG_RT_GROUP_SCHED
- root_task_group.rt_se = (struct sched_rt_entity **)ptr;
+ root_task_group.rt_se = (struct sched_rt_entity **) ptr;
ptr += nr_cpu_ids * sizeof(void **);
- root_task_group.rt_rq = (struct rt_rq **)ptr;
+ root_task_group.rt_rq = (struct rt_rq **) ptr;
ptr += nr_cpu_ids * sizeof(void **);
#endif /* CONFIG_RT_GROUP_SCHED */
}
#ifdef CONFIG_CPUMASK_OFFSTACK
+
for_each_possible_cpu(i) {
- per_cpu(load_balance_mask, i) = (cpumask_var_t)kzalloc_node(
+ per_cpu(load_balance_mask, i) = (cpumask_var_t) kzalloc_node(
cpumask_size(), GFP_KERNEL, cpu_to_node(i));
- per_cpu(select_idle_mask, i) = (cpumask_var_t)kzalloc_node(
+ per_cpu(select_idle_mask, i) = (cpumask_var_t) kzalloc_node(
cpumask_size(), GFP_KERNEL, cpu_to_node(i));
}
#endif /* CONFIG_CPUMASK_OFFSTACK */
init_rt_bandwidth(&def_rt_bandwidth,
- global_rt_period(), global_rt_runtime());
+ global_rt_period(), global_rt_runtime());
init_dl_bandwidth(&def_dl_bandwidth,
- global_rt_period(), global_rt_runtime());
+ global_rt_period(), global_rt_runtime());
#ifdef CONFIG_SMP
init_defrootdomain();
@@ -7863,7 +7979,7 @@ void __init sched_init(void)
#ifdef CONFIG_RT_GROUP_SCHED
init_rt_bandwidth(&root_task_group.rt_bandwidth,
- global_rt_period(), global_rt_runtime());
+ global_rt_period(), global_rt_runtime());
#endif /* CONFIG_RT_GROUP_SCHED */
#ifdef CONFIG_CGROUP_SCHED
@@ -7931,7 +8047,7 @@ void __init sched_init(void)
rq->cpu = i;
rq->online = 0;
rq->idle_stamp = 0;
- rq->avg_idle = 2*sysctl_sched_migration_cost;
+ rq->avg_idle = 2 * sysctl_sched_migration_cost;
rq->max_idle_balance_cost = sysctl_sched_migration_cost;
INIT_LIST_HEAD(&rq->cfs_tasks);
@@ -7983,11 +8099,12 @@ void __init sched_init(void)
}
#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
+
static inline int preempt_count_equals(int preempt_offset)
{
int nested = preempt_count() + sched_rcu_preempt_depth();
- return (nested == preempt_offset);
+ return(nested == preempt_offset);
}
void __might_sleep(const char *file, int line, int preempt_offset)
@@ -7998,11 +8115,11 @@ void __might_sleep(const char *file, int line, int preempt_offset)
* otherwise we will destroy state.
*/
WARN_ONCE(current->state != TASK_RUNNING && current->task_state_change,
- "do not call blocking ops when !TASK_RUNNING; "
- "state=%lx set at [<%p>] %pS\n",
- current->state,
- (void *)current->task_state_change,
- (void *)current->task_state_change);
+ "do not call blocking ops when !TASK_RUNNING; "
+ "state=%lx set at [<%p>] %pS\n",
+ current->state,
+ (void *) current->task_state_change,
+ (void *) current->task_state_change);
___might_sleep(file, line, preempt_offset);
}
@@ -8010,13 +8127,13 @@ EXPORT_SYMBOL(__might_sleep);
void ___might_sleep(const char *file, int line, int preempt_offset)
{
- static unsigned long prev_jiffy; /* ratelimiting */
+ static unsigned long prev_jiffy; /* ratelimiting */
unsigned long preempt_disable_ip;
rcu_sleep_check(); /* WARN_ON_ONCE() by default, no rate limit reqd. */
if ((preempt_count_equals(preempt_offset) && !irqs_disabled() &&
- !is_idle_task(current)) ||
- system_state != SYSTEM_RUNNING || oops_in_progress)
+ !is_idle_task(current)) ||
+ system_state != SYSTEM_RUNNING || oops_in_progress)
return;
if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
return;
@@ -8027,11 +8144,11 @@ void ___might_sleep(const char *file, int line, int preempt_offset)
printk(KERN_ERR
"BUG: sleeping function called from invalid context at %s:%d\n",
- file, line);
+ file, line);
printk(KERN_ERR
"in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n",
- in_atomic(), irqs_disabled(),
- current->pid, current->comm);
+ in_atomic(), irqs_disabled(),
+ current->pid, current->comm);
if (task_stack_end_corrupted(current))
printk(KERN_EMERG "Thread overran stack, or stack corrupted\n");
@@ -8040,7 +8157,7 @@ void ___might_sleep(const char *file, int line, int preempt_offset)
if (irqs_disabled())
print_irqtrace_events(current);
if (IS_ENABLED(CONFIG_DEBUG_PREEMPT)
- && !preempt_count_equals(preempt_offset)) {
+ && !preempt_count_equals(preempt_offset)) {
pr_err("Preemption disabled at:");
print_ip_sym(preempt_disable_ip);
pr_cont("\n");
@@ -8052,6 +8169,7 @@ EXPORT_SYMBOL(___might_sleep);
#endif
#ifdef CONFIG_MAGIC_SYSRQ
+
void normalize_rt_tasks(void)
{
struct task_struct *g, *p;
@@ -8060,6 +8178,7 @@ void normalize_rt_tasks(void)
};
read_lock(&tasklist_lock);
+
for_each_process_thread(g, p) {
/*
* Only normalize user tasks:
@@ -8068,7 +8187,7 @@ void normalize_rt_tasks(void)
continue;
p->se.exec_start = 0;
- schedstat_set(p->se.statistics.wait_start, 0);
+ schedstat_set(p->se.statistics.wait_start, 0);
schedstat_set(p->se.statistics.sleep_start, 0);
schedstat_set(p->se.statistics.block_start, 0);
@@ -8116,6 +8235,7 @@ struct task_struct *curr_task(int cpu)
#endif /* defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) */
#ifdef CONFIG_IA64
+
/**
* set_curr_task - set the current task for a given cpu.
* @cpu: the processor in question.
@@ -8225,7 +8345,7 @@ static void sched_change_group(struct task_struct *tsk, int type)
* to prevent lockdep warnings.
*/
tg = container_of(task_css_check(tsk, cpu_cgrp_id, true),
- struct task_group, css);
+ struct task_group, css);
tg = autogroup_task_group(tsk, tg);
tsk->sched_task_group = tg;
@@ -8341,7 +8461,7 @@ static int tg_rt_schedulable(struct task_group *tg, void *data)
/*
* The sum of our children's runtime should not exceed our own.
*/
- list_for_each_entry_rcu(child, &tg->children, siblings) {
+ list_for_each_entry_rcu(child, &tg->children, siblings) {
period = ktime_to_ns(child->rt_bandwidth.rt_period);
runtime = child->rt_bandwidth.rt_runtime;
@@ -8377,7 +8497,7 @@ static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime)
}
static int tg_set_rt_bandwidth(struct task_group *tg,
- u64 rt_period, u64 rt_runtime)
+ u64 rt_period, u64 rt_runtime)
{
int i, err = 0;
@@ -8422,7 +8542,7 @@ static int sched_group_set_rt_runtime(struct task_group *tg, long rt_runtime_us)
u64 rt_runtime, rt_period;
rt_period = ktime_to_ns(tg->rt_bandwidth.rt_period);
- rt_runtime = (u64)rt_runtime_us * NSEC_PER_USEC;
+ rt_runtime = (u64) rt_runtime_us * NSEC_PER_USEC;
if (rt_runtime_us < 0)
rt_runtime = RUNTIME_INF;
@@ -8462,6 +8582,7 @@ static long sched_group_rt_period(struct task_group *tg)
#endif /* CONFIG_RT_GROUP_SCHED */
#ifdef CONFIG_RT_GROUP_SCHED
+
static int sched_rt_global_constraints(void)
{
int ret = 0;
@@ -8485,12 +8606,14 @@ static int sched_rt_can_attach(struct task_group *tg, struct task_struct *tsk)
}
#else /* !CONFIG_RT_GROUP_SCHED */
+
static int sched_rt_global_constraints(void)
{
unsigned long flags;
int i;
raw_spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags);
+
for_each_possible_cpu(i) {
struct rt_rq *rt_rq = &cpu_rq(i)->rt;
@@ -8587,8 +8710,8 @@ static void sched_rt_do_global(void)
}
int sched_rt_handler(struct ctl_table *table, int write,
- void __user *buffer, size_t *lenp,
- loff_t *ppos)
+ void __user *buffer, size_t *lenp,
+ loff_t *ppos)
{
int old_period, old_runtime;
static DEFINE_MUTEX(mutex);
@@ -8627,8 +8750,8 @@ int sched_rt_handler(struct ctl_table *table, int write,
}
int sched_rr_handler(struct ctl_table *table, int write,
- void __user *buffer, size_t *lenp,
- loff_t *ppos)
+ void __user *buffer, size_t *lenp,
+ loff_t *ppos)
{
int ret;
static DEFINE_MUTEX(mutex);
@@ -8720,7 +8843,8 @@ static int cpu_cgroup_can_attach(struct cgroup_taskset *tset)
struct cgroup_subsys_state *css;
int ret = 0;
- cgroup_taskset_for_each(task, css, tset) {
+ cgroup_taskset_for_each(task, css, tset)
+ {
#ifdef CONFIG_RT_GROUP_SCHED
if (!sched_rt_can_attach(css_tg(css), task))
return -EINVAL;
@@ -8755,18 +8879,19 @@ static void cpu_cgroup_attach(struct cgroup_taskset *tset)
struct cgroup_subsys_state *css;
cgroup_taskset_for_each(task, css, tset)
- sched_move_task(task);
+ sched_move_task(task);
}
#ifdef CONFIG_FAIR_GROUP_SCHED
+
static int cpu_shares_write_u64(struct cgroup_subsys_state *css,
- struct cftype *cftype, u64 shareval)
+ struct cftype *cftype, u64 shareval)
{
return sched_group_set_shares(css_tg(css), scale_load(shareval));
}
static u64 cpu_shares_read_u64(struct cgroup_subsys_state *css,
- struct cftype *cft)
+ struct cftype *cft)
{
struct task_group *tg = css_tg(css);
@@ -8862,7 +8987,7 @@ int tg_set_cfs_quota(struct task_group *tg, long cfs_quota_us)
if (cfs_quota_us < 0)
quota = RUNTIME_INF;
else
- quota = (u64)cfs_quota_us * NSEC_PER_USEC;
+ quota = (u64) cfs_quota_us * NSEC_PER_USEC;
return tg_set_cfs_bandwidth(tg, period, quota);
}
@@ -8884,7 +9009,7 @@ int tg_set_cfs_period(struct task_group *tg, long cfs_period_us)
{
u64 quota, period;
- period = (u64)cfs_period_us * NSEC_PER_USEC;
+ period = (u64) cfs_period_us * NSEC_PER_USEC;
quota = tg->cfs_bandwidth.quota;
return tg_set_cfs_bandwidth(tg, period, quota);
@@ -8901,25 +9026,25 @@ long tg_get_cfs_period(struct task_group *tg)
}
static s64 cpu_cfs_quota_read_s64(struct cgroup_subsys_state *css,
- struct cftype *cft)
+ struct cftype *cft)
{
return tg_get_cfs_quota(css_tg(css));
}
static int cpu_cfs_quota_write_s64(struct cgroup_subsys_state *css,
- struct cftype *cftype, s64 cfs_quota_us)
+ struct cftype *cftype, s64 cfs_quota_us)
{
return tg_set_cfs_quota(css_tg(css), cfs_quota_us);
}
static u64 cpu_cfs_period_read_u64(struct cgroup_subsys_state *css,
- struct cftype *cft)
+ struct cftype *cft)
{
return tg_get_cfs_period(css_tg(css));
}
static int cpu_cfs_period_write_u64(struct cgroup_subsys_state *css,
- struct cftype *cftype, u64 cfs_period_us)
+ struct cftype *cftype, u64 cfs_period_us)
{
return tg_set_cfs_period(css_tg(css), cfs_period_us);
}
@@ -8934,7 +9059,7 @@ struct cfs_schedulable_data {
* note: units are usecs
*/
static u64 normalize_cfs_quota(struct task_group *tg,
- struct cfs_schedulable_data *d)
+ struct cfs_schedulable_data *d)
{
u64 quota, period;
@@ -9017,26 +9142,27 @@ static int cpu_stats_show(struct seq_file *sf, void *v)
#endif /* CONFIG_FAIR_GROUP_SCHED */
#ifdef CONFIG_RT_GROUP_SCHED
+
static int cpu_rt_runtime_write(struct cgroup_subsys_state *css,
- struct cftype *cft, s64 val)
+ struct cftype *cft, s64 val)
{
return sched_group_set_rt_runtime(css_tg(css), val);
}
static s64 cpu_rt_runtime_read(struct cgroup_subsys_state *css,
- struct cftype *cft)
+ struct cftype *cft)
{
return sched_group_rt_runtime(css_tg(css));
}
static int cpu_rt_period_write_uint(struct cgroup_subsys_state *css,
- struct cftype *cftype, u64 rt_period_us)
+ struct cftype *cftype, u64 rt_period_us)
{
return sched_group_set_rt_period(css_tg(css), rt_period_us);
}
static u64 cpu_rt_period_read_uint(struct cgroup_subsys_state *css,
- struct cftype *cft)
+ struct cftype *cft)
{
return sched_group_rt_period(css_tg(css));
}
@@ -9078,22 +9204,22 @@ static struct cftype cpu_files[] = {
.write_u64 = cpu_rt_period_write_uint,
},
#endif
- { } /* terminate */
+ {} /* terminate */
};
struct cgroup_subsys cpu_cgrp_subsys = {
- .css_alloc = cpu_cgroup_css_alloc,
- .css_online = cpu_cgroup_css_online,
- .css_released = cpu_cgroup_css_released,
- .css_free = cpu_cgroup_css_free,
- .fork = cpu_cgroup_fork,
- .can_attach = cpu_cgroup_can_attach,
- .attach = cpu_cgroup_attach,
- .legacy_cftypes = cpu_files,
- .early_init = true,
+ .css_alloc = cpu_cgroup_css_alloc,
+ .css_online = cpu_cgroup_css_online,
+ .css_released = cpu_cgroup_css_released,
+ .css_free = cpu_cgroup_css_free,
+ .fork = cpu_cgroup_fork,
+ .can_attach = cpu_cgroup_can_attach,
+ .attach = cpu_cgroup_attach,
+ .legacy_cftypes = cpu_files,
+ .early_init = true,
};
-#endif /* CONFIG_CGROUP_SCHED */
+#endif /* CONFIG_CGROUP_SCHED */
void dump_cpu_task(int cpu)
{
@@ -9114,14 +9240,14 @@ void dump_cpu_task(int cpu)
* the relative distance between them is ~25%.)
*/
const int sched_prio_to_weight[40] = {
- /* -20 */ 88761, 71755, 56483, 46273, 36291,
- /* -15 */ 29154, 23254, 18705, 14949, 11916,
- /* -10 */ 9548, 7620, 6100, 4904, 3906,
- /* -5 */ 3121, 2501, 1991, 1586, 1277,
- /* 0 */ 1024, 820, 655, 526, 423,
- /* 5 */ 335, 272, 215, 172, 137,
- /* 10 */ 110, 87, 70, 56, 45,
- /* 15 */ 36, 29, 23, 18, 15,
+ /* -20 */ 88761, 71755, 56483, 46273, 36291,
+ /* -15 */ 29154, 23254, 18705, 14949, 11916,
+ /* -10 */ 9548, 7620, 6100, 4904, 3906,
+ /* -5 */ 3121, 2501, 1991, 1586, 1277,
+ /* 0 */ 1024, 820, 655, 526, 423,
+ /* 5 */ 335, 272, 215, 172, 137,
+ /* 10 */ 110, 87, 70, 56, 45,
+ /* 15 */ 36, 29, 23, 18, 15,
};
/*
@@ -9132,12 +9258,12 @@ const int sched_prio_to_weight[40] = {
* into multiplications:
*/
const u32 sched_prio_to_wmult[40] = {
- /* -20 */ 48388, 59856, 76040, 92818, 118348,
- /* -15 */ 147320, 184698, 229616, 287308, 360437,
- /* -10 */ 449829, 563644, 704093, 875809, 1099582,
- /* -5 */ 1376151, 1717300, 2157191, 2708050, 3363326,
- /* 0 */ 4194304, 5237765, 6557202, 8165337, 10153587,
- /* 5 */ 12820798, 15790321, 19976592, 24970740, 31350126,
- /* 10 */ 39045157, 49367440, 61356676, 76695844, 95443717,
- /* 15 */ 119304647, 148102320, 186737708, 238609294, 286331153,
+ /* -20 */ 48388, 59856, 76040, 92818, 118348,
+ /* -15 */ 147320, 184698, 229616, 287308, 360437,
+ /* -10 */ 449829, 563644, 704093, 875809, 1099582,
+ /* -5 */ 1376151, 1717300, 2157191, 2708050, 3363326,
+ /* 0 */ 4194304, 5237765, 6557202, 8165337, 10153587,
+ /* 5 */ 12820798, 15790321, 19976592, 24970740, 31350126,
+ /* 10 */ 39045157, 49367440, 61356676, 76695844, 95443717,
+ /* 15 */ 119304647, 148102320, 186737708, 238609294, 286331153,
};
diff --git a/kernel/sched/cpudeadline.c b/kernel/sched/cpudeadline.c
index e731190..fba235c 100644
--- a/kernel/sched/cpudeadline.c
+++ b/kernel/sched/cpudeadline.c
@@ -128,10 +128,10 @@ int cpudl_find(struct cpudl *cp, struct task_struct *p,
const struct sched_dl_entity *dl_se = &p->dl;
if (later_mask &&
- cpumask_and(later_mask, cp->free_cpus, tsk_cpus_allowed(p))) {
+ cpumask_and(later_mask, cp->free_cpus, &p->cpus_allowed)) {
best_cpu = cpumask_any(later_mask);
goto out;
- } else if (cpumask_test_cpu(cpudl_maximum(cp), tsk_cpus_allowed(p)) &&
+ } else if (cpumask_test_cpu(cpudl_maximum(cp), &p->cpus_allowed) &&
dl_time_before(dl_se->deadline, cp->elements[0].dl)) {
best_cpu = cpudl_maximum(cp);
if (later_mask)
diff --git a/kernel/sched/cpupri.c b/kernel/sched/cpupri.c
index 11e9705..981fcd7 100644
--- a/kernel/sched/cpupri.c
+++ b/kernel/sched/cpupri.c
@@ -103,11 +103,11 @@ int cpupri_find(struct cpupri *cp, struct task_struct *p,
if (skip)
continue;
- if (cpumask_any_and(tsk_cpus_allowed(p), vec->mask) >= nr_cpu_ids)
+ if (cpumask_any_and(&p->cpus_allowed, vec->mask) >= nr_cpu_ids)
continue;
if (lowest_mask) {
- cpumask_and(lowest_mask, tsk_cpus_allowed(p), vec->mask);
+ cpumask_and(lowest_mask, &p->cpus_allowed, vec->mask);
/*
* We have to ensure that we have at least one bit
diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c
index e00accf..98520aa 100644
--- a/kernel/sched/deadline.c
+++ b/kernel/sched/deadline.c
@@ -134,7 +134,7 @@ static void inc_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)
{
struct task_struct *p = dl_task_of(dl_se);
- if (tsk_nr_cpus_allowed(p) > 1)
+ if (p->nr_cpus_allowed > 1)
dl_rq->dl_nr_migratory++;
update_dl_migration(dl_rq);
@@ -144,7 +144,7 @@ static void dec_dl_migration(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq)
{
struct task_struct *p = dl_task_of(dl_se);
- if (tsk_nr_cpus_allowed(p) > 1)
+ if (p->nr_cpus_allowed > 1)
dl_rq->dl_nr_migratory--;
update_dl_migration(dl_rq);
@@ -959,7 +959,7 @@ static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags)
enqueue_dl_entity(&p->dl, pi_se, flags);
- if (!task_current(rq, p) && tsk_nr_cpus_allowed(p) > 1)
+ if (!task_current(rq, p) && p->nr_cpus_allowed > 1)
enqueue_pushable_dl_task(rq, p);
}
@@ -1033,9 +1033,9 @@ select_task_rq_dl(struct task_struct *p, int cpu, int sd_flag, int flags)
* try to make it stay here, it might be important.
*/
if (unlikely(dl_task(curr)) &&
- (tsk_nr_cpus_allowed(curr) < 2 ||
+ (curr->nr_cpus_allowed < 2 ||
!dl_entity_preempt(&p->dl, &curr->dl)) &&
- (tsk_nr_cpus_allowed(p) > 1)) {
+ (p->nr_cpus_allowed > 1)) {
int target = find_later_rq(p);
if (target != -1 &&
@@ -1056,7 +1056,7 @@ static void check_preempt_equal_dl(struct rq *rq, struct task_struct *p)
* Current can't be migrated, useless to reschedule,
* let's hope p can move out.
*/
- if (tsk_nr_cpus_allowed(rq->curr) == 1 ||
+ if (rq->curr->nr_cpus_allowed == 1 ||
cpudl_find(&rq->rd->cpudl, rq->curr, NULL) == -1)
return;
@@ -1064,7 +1064,7 @@ static void check_preempt_equal_dl(struct rq *rq, struct task_struct *p)
* p is migratable, so let's not schedule it and
* see if it is pushed or pulled somewhere else.
*/
- if (tsk_nr_cpus_allowed(p) != 1 &&
+ if (p->nr_cpus_allowed != 1 &&
cpudl_find(&rq->rd->cpudl, p, NULL) != -1)
return;
@@ -1179,7 +1179,7 @@ static void put_prev_task_dl(struct rq *rq, struct task_struct *p)
{
update_curr_dl(rq);
- if (on_dl_rq(&p->dl) && tsk_nr_cpus_allowed(p) > 1)
+ if (on_dl_rq(&p->dl) && p->nr_cpus_allowed > 1)
enqueue_pushable_dl_task(rq, p);
}
@@ -1280,7 +1280,7 @@ static int find_later_rq(struct task_struct *task)
if (unlikely(!later_mask))
return -1;
- if (tsk_nr_cpus_allowed(task) == 1)
+ if (task->nr_cpus_allowed == 1)
return -1;
/*
@@ -1386,7 +1386,7 @@ static struct rq *find_lock_later_rq(struct task_struct *task, struct rq *rq)
if (double_lock_balance(rq, later_rq)) {
if (unlikely(task_rq(task) != rq ||
!cpumask_test_cpu(later_rq->cpu,
- tsk_cpus_allowed(task)) ||
+ &task->cpus_allowed) ||
task_running(rq, task) ||
!dl_task(task) ||
!task_on_rq_queued(task))) {
@@ -1426,7 +1426,7 @@ static struct task_struct *pick_next_pushable_dl_task(struct rq *rq)
BUG_ON(rq->cpu != task_cpu(p));
BUG_ON(task_current(rq, p));
- BUG_ON(tsk_nr_cpus_allowed(p) <= 1);
+ BUG_ON(p->nr_cpus_allowed <= 1);
BUG_ON(!task_on_rq_queued(p));
BUG_ON(!dl_task(p));
@@ -1465,7 +1465,7 @@ static int push_dl_task(struct rq *rq)
*/
if (dl_task(rq->curr) &&
dl_time_before(next_task->dl.deadline, rq->curr->dl.deadline) &&
- tsk_nr_cpus_allowed(rq->curr) > 1) {
+ rq->curr->nr_cpus_allowed > 1) {
resched_curr(rq);
return 0;
}
@@ -1612,9 +1612,9 @@ static void task_woken_dl(struct rq *rq, struct task_struct *p)
{
if (!task_running(rq, p) &&
!test_tsk_need_resched(rq->curr) &&
- tsk_nr_cpus_allowed(p) > 1 &&
+ p->nr_cpus_allowed > 1 &&
dl_task(rq->curr) &&
- (tsk_nr_cpus_allowed(rq->curr) < 2 ||
+ (rq->curr->nr_cpus_allowed < 2 ||
!dl_entity_preempt(&p->dl, &rq->curr->dl))) {
push_dl_tasks(rq);
}
@@ -1728,7 +1728,7 @@ static void switched_to_dl(struct rq *rq, struct task_struct *p)
if (rq->curr != p) {
#ifdef CONFIG_SMP
- if (tsk_nr_cpus_allowed(p) > 1 && rq->dl.overloaded)
+ if (p->nr_cpus_allowed > 1 && rq->dl.overloaded)
queue_push_tasks(rq);
#endif
if (dl_task(rq->curr))
diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index b0691f4..32be096 100644
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -336,7 +336,7 @@ static void inc_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
rt_rq = &rq_of_rt_rq(rt_rq)->rt;
rt_rq->rt_nr_total++;
- if (tsk_nr_cpus_allowed(p) > 1)
+ if (p->nr_cpus_allowed > 1)
rt_rq->rt_nr_migratory++;
update_rt_migration(rt_rq);
@@ -353,7 +353,7 @@ static void dec_rt_migration(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
rt_rq = &rq_of_rt_rq(rt_rq)->rt;
rt_rq->rt_nr_total--;
- if (tsk_nr_cpus_allowed(p) > 1)
+ if (p->nr_cpus_allowed > 1)
rt_rq->rt_nr_migratory--;
update_rt_migration(rt_rq);
@@ -1325,7 +1325,7 @@ enqueue_task_rt(struct rq *rq, struct task_struct *p, int flags)
enqueue_rt_entity(rt_se, flags);
- if (!task_current(rq, p) && tsk_nr_cpus_allowed(p) > 1)
+ if (!task_current(rq, p) && p->nr_cpus_allowed > 1)
enqueue_pushable_task(rq, p);
}
@@ -1414,7 +1414,7 @@ select_task_rq_rt(struct task_struct *p, int cpu, int sd_flag, int flags)
* will have to sort it out.
*/
if (curr && unlikely(rt_task(curr)) &&
- (tsk_nr_cpus_allowed(curr) < 2 ||
+ (curr->nr_cpus_allowed < 2 ||
curr->prio <= p->prio)) {
int target = find_lowest_rq(p);
@@ -1438,7 +1438,7 @@ static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p)
* Current can't be migrated, useless to reschedule,
* let's hope p can move out.
*/
- if (tsk_nr_cpus_allowed(rq->curr) == 1 ||
+ if (rq->curr->nr_cpus_allowed == 1 ||
!cpupri_find(&rq->rd->cpupri, rq->curr, NULL))
return;
@@ -1446,7 +1446,7 @@ static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p)
* p is migratable, so let's not schedule it and
* see if it is pushed or pulled somewhere else.
*/
- if (tsk_nr_cpus_allowed(p) != 1
+ if (p->nr_cpus_allowed != 1
&& cpupri_find(&rq->rd->cpupri, p, NULL))
return;
@@ -1580,7 +1580,7 @@ static void put_prev_task_rt(struct rq *rq, struct task_struct *p)
* The previous task needs to be made eligible for pushing
* if it is still active
*/
- if (on_rt_rq(&p->rt) && tsk_nr_cpus_allowed(p) > 1)
+ if (on_rt_rq(&p->rt) && p->nr_cpus_allowed > 1)
enqueue_pushable_task(rq, p);
}
@@ -1630,7 +1630,7 @@ static int find_lowest_rq(struct task_struct *task)
if (unlikely(!lowest_mask))
return -1;
- if (tsk_nr_cpus_allowed(task) == 1)
+ if (task->nr_cpus_allowed == 1)
return -1; /* No other targets possible */
if (!cpupri_find(&task_rq(task)->rd->cpupri, task, lowest_mask))
@@ -1763,7 +1763,7 @@ static struct task_struct *pick_next_pushable_task(struct rq *rq)
BUG_ON(rq->cpu != task_cpu(p));
BUG_ON(task_current(rq, p));
- BUG_ON(tsk_nr_cpus_allowed(p) <= 1);
+ BUG_ON(p->nr_cpus_allowed <= 1);
BUG_ON(!task_on_rq_queued(p));
BUG_ON(!rt_task(p));
@@ -2123,9 +2123,9 @@ static void task_woken_rt(struct rq *rq, struct task_struct *p)
{
if (!task_running(rq, p) &&
!test_tsk_need_resched(rq->curr) &&
- tsk_nr_cpus_allowed(p) > 1 &&
+ p->nr_cpus_allowed > 1 &&
(dl_task(rq->curr) || rt_task(rq->curr)) &&
- (tsk_nr_cpus_allowed(rq->curr) < 2 ||
+ (rq->curr->nr_cpus_allowed < 2 ||
rq->curr->prio <= p->prio))
push_rt_tasks(rq);
}
--
2.7.4
More information about the linux-yocto
mailing list