mirror of
https://github.com/team-infusion-developers/android_kernel_samsung_msm8976.git
synced 2024-09-21 20:04:01 +00:00
sched: add sync wakeup recognition in select_best_cpu
If a wakeup is a sync wakeup, we need to discount the currently running task's load from the waker's CPU as we calculate the best CPU for the waking task to land on. Change-Id: I00c5df626d17868323d60fb90b4513c0dd314825 Signed-off-by: Steve Muckle <smuckle@codeaurora.org>
This commit is contained in:
parent
6e778f0cdc
commit
ee9ddb5f3c
1 changed files with 73 additions and 26 deletions
|
@ -1487,10 +1487,34 @@ static inline u64 cpu_load(int cpu)
|
|||
return scale_load_to_cpu(rq->cumulative_runnable_avg, cpu);
|
||||
}
|
||||
|
||||
static int
|
||||
spill_threshold_crossed(struct task_struct *p, struct rq *rq, int cpu)
|
||||
static inline u64 cpu_load_sync(int cpu, int sync)
|
||||
{
|
||||
u64 total_load = cpu_load(cpu) + scale_load_to_cpu(task_load(p), cpu);
|
||||
struct rq *rq = cpu_rq(cpu);
|
||||
u64 load;
|
||||
|
||||
load = rq->cumulative_runnable_avg;
|
||||
|
||||
/*
|
||||
* If load is being checked in a sync wakeup environment,
|
||||
* we may want to discount the load of the currently running
|
||||
* task.
|
||||
*/
|
||||
if (sync && cpu == smp_processor_id()) {
|
||||
if (load > rq->curr->ravg.demand)
|
||||
load -= rq->curr->ravg.demand;
|
||||
else
|
||||
load = 0;
|
||||
}
|
||||
|
||||
return scale_load_to_cpu(load, cpu);
|
||||
}
|
||||
|
||||
static int
|
||||
spill_threshold_crossed(struct task_struct *p, struct rq *rq, int cpu,
|
||||
int sync)
|
||||
{
|
||||
u64 total_load = cpu_load_sync(cpu, sync) +
|
||||
scale_load_to_cpu(task_load(p), cpu);
|
||||
|
||||
if (total_load > sched_spill_load ||
|
||||
(rq->nr_running + 1) > sysctl_sched_spill_nr_run)
|
||||
|
@ -1508,6 +1532,26 @@ int mostly_idle_cpu(int cpu)
|
|||
&& !sched_cpu_high_irqload(cpu);
|
||||
}
|
||||
|
||||
static int mostly_idle_cpu_sync(int cpu, int sync)
|
||||
{
|
||||
struct rq *rq = cpu_rq(cpu);
|
||||
u64 load = cpu_load_sync(cpu, sync);
|
||||
int nr_running;
|
||||
|
||||
nr_running = rq->nr_running;
|
||||
|
||||
/*
|
||||
* Sync wakeups mean that the waker task will go to sleep
|
||||
* soon so we should discount its load from this test.
|
||||
*/
|
||||
if (sync && cpu == smp_processor_id())
|
||||
nr_running--;
|
||||
|
||||
return load <= rq->mostly_idle_load &&
|
||||
nr_running <= rq->mostly_idle_nr_run &&
|
||||
!sched_cpu_high_irqload(cpu);
|
||||
}
|
||||
|
||||
static int boost_refcount;
|
||||
static DEFINE_SPINLOCK(boost_lock);
|
||||
static DEFINE_MUTEX(boost_mutex);
|
||||
|
@ -1622,15 +1666,15 @@ static int task_will_fit(struct task_struct *p, int cpu)
|
|||
return 0;
|
||||
}
|
||||
|
||||
static int eligible_cpu(struct task_struct *p, int cpu)
|
||||
static int eligible_cpu(struct task_struct *p, int cpu, int sync)
|
||||
{
|
||||
struct rq *rq = cpu_rq(cpu);
|
||||
|
||||
if (mostly_idle_cpu(cpu))
|
||||
if (mostly_idle_cpu_sync(cpu, sync))
|
||||
return 1;
|
||||
|
||||
if (rq->capacity != max_capacity)
|
||||
return !spill_threshold_crossed(p, rq, cpu);
|
||||
return !spill_threshold_crossed(p, rq, cpu, sync);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
@ -1706,7 +1750,7 @@ static unsigned int power_cost(struct task_struct *p, int cpu)
|
|||
return power_cost_at_freq(cpu, task_freq);
|
||||
}
|
||||
|
||||
static int best_small_task_cpu(struct task_struct *p)
|
||||
static int best_small_task_cpu(struct task_struct *p, int sync)
|
||||
{
|
||||
int best_busy_cpu = -1, best_fallback_cpu = -1;
|
||||
int min_cost_cpu = -1, min_cstate_cpu = -1;
|
||||
|
@ -1728,8 +1772,8 @@ static int best_small_task_cpu(struct task_struct *p)
|
|||
for_each_cpu(i, &search_cpus) {
|
||||
|
||||
trace_sched_cpu_load(cpu_rq(i), idle_cpu(i),
|
||||
mostly_idle_cpu(i), sched_irqload(i),
|
||||
power_cost(p, i));
|
||||
mostly_idle_cpu_sync(i, sync),
|
||||
sched_irqload(i), power_cost(p, i));
|
||||
|
||||
cpu_cost = power_cost(p, i);
|
||||
if (cpu_cost < min_cost) {
|
||||
|
@ -1743,7 +1787,8 @@ static int best_small_task_cpu(struct task_struct *p)
|
|||
/* Optimization to steer task towards the minimum power
|
||||
cost CPU. The tradeoff is that we may have to check
|
||||
the same information again in pass 2 */
|
||||
if (!cpu_rq(min_cost_cpu)->cstate && mostly_idle_cpu(min_cost_cpu))
|
||||
if (!cpu_rq(min_cost_cpu)->cstate &&
|
||||
mostly_idle_cpu_sync(min_cost_cpu, sync))
|
||||
return min_cost_cpu;
|
||||
|
||||
for_each_cpu(i, &search_cpus) {
|
||||
|
@ -1766,11 +1811,11 @@ static int best_small_task_cpu(struct task_struct *p)
|
|||
continue;
|
||||
}
|
||||
|
||||
if (mostly_idle_cpu(i))
|
||||
if (mostly_idle_cpu_sync(i, sync))
|
||||
return i;
|
||||
|
||||
load = cpu_load(i);
|
||||
if (!spill_threshold_crossed(p, rq, i)) {
|
||||
load = cpu_load_sync(i, sync);
|
||||
if (!spill_threshold_crossed(p, rq, i, sync)) {
|
||||
if (load < min_busy_load) {
|
||||
min_busy_load = load;
|
||||
best_busy_cpu = i;
|
||||
|
@ -1857,7 +1902,8 @@ static int select_packing_target(struct task_struct *p, int best_cpu)
|
|||
|
||||
|
||||
/* return cheapest cpu that can fit this task */
|
||||
static int select_best_cpu(struct task_struct *p, int target, int reason)
|
||||
static int select_best_cpu(struct task_struct *p, int target, int reason,
|
||||
int sync)
|
||||
{
|
||||
int i, best_cpu = -1, fallback_idle_cpu = -1, min_cstate_cpu = -1;
|
||||
int prev_cpu = task_cpu(p);
|
||||
|
@ -1872,7 +1918,7 @@ static int select_best_cpu(struct task_struct *p, int target, int reason)
|
|||
trace_sched_task_load(p, small_task, boost, reason);
|
||||
|
||||
if (small_task && !boost) {
|
||||
best_cpu = best_small_task_cpu(p);
|
||||
best_cpu = best_small_task_cpu(p, sync);
|
||||
goto done;
|
||||
}
|
||||
|
||||
|
@ -1880,8 +1926,8 @@ static int select_best_cpu(struct task_struct *p, int target, int reason)
|
|||
for_each_cpu_and(i, tsk_cpus_allowed(p), cpu_online_mask) {
|
||||
|
||||
trace_sched_cpu_load(cpu_rq(i), idle_cpu(i),
|
||||
mostly_idle_cpu(i), sched_irqload(i),
|
||||
power_cost(p, i));
|
||||
mostly_idle_cpu_sync(i, sync),
|
||||
sched_irqload(i), power_cost(p, i));
|
||||
|
||||
if (skip_cpu(p, i, reason))
|
||||
continue;
|
||||
|
@ -1892,8 +1938,8 @@ static int select_best_cpu(struct task_struct *p, int target, int reason)
|
|||
* where the task will fit.
|
||||
*/
|
||||
if (!task_will_fit(p, i)) {
|
||||
if (mostly_idle_cpu(i)) {
|
||||
load = cpu_load(i);
|
||||
if (mostly_idle_cpu_sync(i, sync)) {
|
||||
load = cpu_load_sync(i, sync);
|
||||
if (load < min_fallback_load) {
|
||||
min_fallback_load = load;
|
||||
fallback_idle_cpu = i;
|
||||
|
@ -1902,7 +1948,7 @@ static int select_best_cpu(struct task_struct *p, int target, int reason)
|
|||
continue;
|
||||
}
|
||||
|
||||
if (!eligible_cpu(p, i))
|
||||
if (!eligible_cpu(p, i, sync))
|
||||
continue;
|
||||
|
||||
/*
|
||||
|
@ -1911,7 +1957,7 @@ static int select_best_cpu(struct task_struct *p, int target, int reason)
|
|||
* spill.
|
||||
*/
|
||||
|
||||
load = cpu_load(i);
|
||||
load = cpu_load_sync(i, sync);
|
||||
cpu_cost = power_cost(p, i);
|
||||
cstate = cpu_rq(i)->cstate;
|
||||
|
||||
|
@ -1979,7 +2025,7 @@ static int select_best_cpu(struct task_struct *p, int target, int reason)
|
|||
}
|
||||
|
||||
if (min_cstate_cpu >= 0 && (prefer_idle ||
|
||||
!(best_cpu >= 0 && mostly_idle_cpu(best_cpu))))
|
||||
!(best_cpu >= 0 && mostly_idle_cpu_sync(best_cpu, sync))))
|
||||
best_cpu = min_cstate_cpu;
|
||||
done:
|
||||
if (best_cpu < 0) {
|
||||
|
@ -2325,7 +2371,7 @@ void check_for_migration(struct rq *rq, struct task_struct *p)
|
|||
return;
|
||||
|
||||
raw_spin_lock(&migration_lock);
|
||||
new_cpu = select_best_cpu(p, cpu, reason);
|
||||
new_cpu = select_best_cpu(p, cpu, reason, 0);
|
||||
|
||||
if (new_cpu != cpu) {
|
||||
active_balance = kick_active_balance(rq, p, new_cpu);
|
||||
|
@ -2354,7 +2400,8 @@ static inline int task_will_fit(struct task_struct *p, int cpu)
|
|||
return 1;
|
||||
}
|
||||
|
||||
static inline int select_best_cpu(struct task_struct *p, int target, int reason)
|
||||
static inline int select_best_cpu(struct task_struct *p, int target,
|
||||
int reason, int sync)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
@ -2370,7 +2417,7 @@ static inline int power_cost(struct task_struct *p, int cpu)
|
|||
}
|
||||
|
||||
static inline int
|
||||
spill_threshold_crossed(struct task_struct *p, struct rq *rq, int cpu)
|
||||
spill_threshold_crossed(struct task_struct *p, struct rq *rq, int cpu, int sync)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
@ -4701,7 +4748,7 @@ select_task_rq_fair(struct task_struct *p, int sd_flag, int wake_flags)
|
|||
return prev_cpu;
|
||||
|
||||
if (sched_enable_hmp)
|
||||
return select_best_cpu(p, prev_cpu, 0);
|
||||
return select_best_cpu(p, prev_cpu, 0, sync);
|
||||
|
||||
if (sd_flag & SD_BALANCE_WAKE) {
|
||||
if (cpumask_test_cpu(cpu, tsk_cpus_allowed(p)))
|
||||
|
|
Loading…
Reference in a new issue