Piteball/android_kernel_samsung_msm8976
1
1
Fork 0
mirror of https://github.com/team-infusion-developers/android_kernel_samsung_msm8976.git synced 2024-09-21 20:04:01 +00:00
Code Issues Projects Releases Wiki Activity

sched: add sync wakeup recognition in select_best_cpu

Browse source 
If a wakeup is a sync wakeup, we need to discount the currently
running task's load from the waker's CPU as we calculate the best
CPU for the waking task to land on.

Change-Id: I00c5df626d17868323d60fb90b4513c0dd314825
Signed-off-by: Steve Muckle <smuckle@codeaurora.org>
This commit is contained in:
Steve Muckle 2014-11-20 17:46:37 -08:00
parent 6e778f0cdc
commit ee9ddb5f3c
1 changed files with 73 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

99
kernel/sched/fair.c
View file

@ -1487,10 +1487,34 @@ static inline u64 cpu_load(int cpu)
return scale_load_to_cpu(rq->cumulative_runnable_avg, cpu);
}
static int
spill_threshold_crossed(struct task_struct *p, struct rq *rq, int cpu)
static inline u64 cpu_load_sync(int cpu, int sync)
{
u64 total_load = cpu_load(cpu) + scale_load_to_cpu(task_load(p), cpu);
struct rq *rq = cpu_rq(cpu);
u64 load;
load = rq->cumulative_runnable_avg;
/*
* If load is being checked in a sync wakeup environment,
* we may want to discount the load of the currently running
* task.
*/
if (sync && cpu == smp_processor_id()) {
if (load > rq->curr->ravg.demand)
load -= rq->curr->ravg.demand;
else
load = 0;
}
return scale_load_to_cpu(load, cpu);
}
static int
spill_threshold_crossed(struct task_struct *p, struct rq *rq, int cpu,
int sync)
{
u64 total_load = cpu_load_sync(cpu, sync) +
scale_load_to_cpu(task_load(p), cpu);
if (total_load > sched_spill_load ||
(rq->nr_running + 1) > sysctl_sched_spill_nr_run)
@ -1508,6 +1532,26 @@ int mostly_idle_cpu(int cpu)
&& !sched_cpu_high_irqload(cpu);
}
static int mostly_idle_cpu_sync(int cpu, int sync)
{
struct rq *rq = cpu_rq(cpu);
u64 load = cpu_load_sync(cpu, sync);
int nr_running;
nr_running = rq->nr_running;
/*
* Sync wakeups mean that the waker task will go to sleep
* soon so we should discount its load from this test.
*/
if (sync && cpu == smp_processor_id())
nr_running--;
return load <= rq->mostly_idle_load &&
nr_running <= rq->mostly_idle_nr_run &&
!sched_cpu_high_irqload(cpu);
}
static int boost_refcount;
static DEFINE_SPINLOCK(boost_lock);
static DEFINE_MUTEX(boost_mutex);
@ -1622,15 +1666,15 @@ static int task_will_fit(struct task_struct *p, int cpu)
return 0;
}
static int eligible_cpu(struct task_struct *p, int cpu)
static int eligible_cpu(struct task_struct *p, int cpu, int sync)
{
struct rq *rq = cpu_rq(cpu);
if (mostly_idle_cpu(cpu))
if (mostly_idle_cpu_sync(cpu, sync))
return 1;
if (rq->capacity != max_capacity)
return !spill_threshold_crossed(p, rq, cpu);
return !spill_threshold_crossed(p, rq, cpu, sync);
return 0;
}
@ -1706,7 +1750,7 @@ static unsigned int power_cost(struct task_struct *p, int cpu)
return power_cost_at_freq(cpu, task_freq);
}
static int best_small_task_cpu(struct task_struct *p)
static int best_small_task_cpu(struct task_struct *p, int sync)
{
int best_busy_cpu = -1, best_fallback_cpu = -1;
int min_cost_cpu = -1, min_cstate_cpu = -1;
@ -1728,8 +1772,8 @@ static int best_small_task_cpu(struct task_struct *p)
for_each_cpu(i, &search_cpus) {
trace_sched_cpu_load(cpu_rq(i), idle_cpu(i),
mostly_idle_cpu(i), sched_irqload(i),
power_cost(p, i));
mostly_idle_cpu_sync(i, sync),
sched_irqload(i), power_cost(p, i));
cpu_cost = power_cost(p, i);
if (cpu_cost < min_cost) {
@ -1743,7 +1787,8 @@ static int best_small_task_cpu(struct task_struct *p)
/* Optimization to steer task towards the minimum power
cost CPU. The tradeoff is that we may have to check
the same information again in pass 2 */
if (!cpu_rq(min_cost_cpu)->cstate && mostly_idle_cpu(min_cost_cpu))
if (!cpu_rq(min_cost_cpu)->cstate &&
mostly_idle_cpu_sync(min_cost_cpu, sync))
return min_cost_cpu;
for_each_cpu(i, &search_cpus) {
@ -1766,11 +1811,11 @@ static int best_small_task_cpu(struct task_struct *p)
continue;
}
if (mostly_idle_cpu(i))
if (mostly_idle_cpu_sync(i, sync))
return i;
load = cpu_load(i);
if (!spill_threshold_crossed(p, rq, i)) {
load = cpu_load_sync(i, sync);
if (!spill_threshold_crossed(p, rq, i, sync)) {
if (load < min_busy_load) {
min_busy_load = load;
best_busy_cpu = i;
@ -1857,7 +1902,8 @@ static int select_packing_target(struct task_struct *p, int best_cpu)
/* return cheapest cpu that can fit this task */
static int select_best_cpu(struct task_struct *p, int target, int reason)
static int select_best_cpu(struct task_struct *p, int target, int reason,
int sync)
{
int i, best_cpu = -1, fallback_idle_cpu = -1, min_cstate_cpu = -1;
int prev_cpu = task_cpu(p);
@ -1872,7 +1918,7 @@ static int select_best_cpu(struct task_struct *p, int target, int reason)
trace_sched_task_load(p, small_task, boost, reason);
if (small_task && !boost) {
best_cpu = best_small_task_cpu(p);
best_cpu = best_small_task_cpu(p, sync);
goto done;
}
@ -1880,8 +1926,8 @@ static int select_best_cpu(struct task_struct *p, int target, int reason)
for_each_cpu_and(i, tsk_cpus_allowed(p), cpu_online_mask) {
trace_sched_cpu_load(cpu_rq(i), idle_cpu(i),
mostly_idle_cpu(i), sched_irqload(i),
power_cost(p, i));
mostly_idle_cpu_sync(i, sync),
sched_irqload(i), power_cost(p, i));
if (skip_cpu(p, i, reason))
continue;
@ -1892,8 +1938,8 @@ static int select_best_cpu(struct task_struct *p, int target, int reason)
* where the task will fit.
*/
if (!task_will_fit(p, i)) {
if (mostly_idle_cpu(i)) {
load = cpu_load(i);
if (mostly_idle_cpu_sync(i, sync)) {
load = cpu_load_sync(i, sync);
if (load < min_fallback_load) {
min_fallback_load = load;
fallback_idle_cpu = i;
@ -1902,7 +1948,7 @@ static int select_best_cpu(struct task_struct *p, int target, int reason)
continue;
}
if (!eligible_cpu(p, i))
if (!eligible_cpu(p, i, sync))
continue;
/*
@ -1911,7 +1957,7 @@ static int select_best_cpu(struct task_struct *p, int target, int reason)
* spill.
*/
load = cpu_load(i);
load = cpu_load_sync(i, sync);
cpu_cost = power_cost(p, i);
cstate = cpu_rq(i)->cstate;
@ -1979,7 +2025,7 @@ static int select_best_cpu(struct task_struct *p, int target, int reason)
}
if (min_cstate_cpu >= 0 && (prefer_idle ||
!(best_cpu >= 0 && mostly_idle_cpu(best_cpu))))
!(best_cpu >= 0 && mostly_idle_cpu_sync(best_cpu, sync))))
best_cpu = min_cstate_cpu;
done:
if (best_cpu < 0) {
@ -2325,7 +2371,7 @@ void check_for_migration(struct rq *rq, struct task_struct *p)
return;
raw_spin_lock(&migration_lock);
new_cpu = select_best_cpu(p, cpu, reason);
new_cpu = select_best_cpu(p, cpu, reason, 0);
if (new_cpu != cpu) {
active_balance = kick_active_balance(rq, p, new_cpu);
@ -2354,7 +2400,8 @@ static inline int task_will_fit(struct task_struct *p, int cpu)
return 1;
}
static inline int select_best_cpu(struct task_struct *p, int target, int reason)
static inline int select_best_cpu(struct task_struct *p, int target,
int reason, int sync)
{
return 0;
}
@ -2370,7 +2417,7 @@ static inline int power_cost(struct task_struct *p, int cpu)
}
static inline int
spill_threshold_crossed(struct task_struct *p, struct rq *rq, int cpu)
spill_threshold_crossed(struct task_struct *p, struct rq *rq, int cpu, int sync)
{
return 0;
}
@ -4701,7 +4748,7 @@ select_task_rq_fair(struct task_struct *p, int sd_flag, int wake_flags)
return prev_cpu;
if (sched_enable_hmp)
return select_best_cpu(p, prev_cpu, 0);
return select_best_cpu(p, prev_cpu, 0, sync);
if (sd_flag & SD_BALANCE_WAKE) {
if (cpumask_test_cpu(cpu, tsk_cpus_allowed(p)))