Skip to content

Commit 4c2a204

Browse files
johnstultz-workPeter Zijlstra
authored andcommitted
sched: Add is_blocked task flag
Add a new is_blocked flag to the task struct. This flag is set by try_to_block_task() and cleared by ttwu_do_wakeup() and tracks if the task is blocked. Traditionally this would mirror !p->on_rq, however due things like DELAY_DEQUEUE and PROXY_EXEC, this can diverge, so its useful to manage separately. Additionally with this, we might be able to get rid of the p->se.sched_delayed (ab)use in the core code (eventually). Taken whole cloth from Peter's email: https://lore.kernel.org/lkml/20260501132143.GC1026330@noisy.programming.kicks-ass.net/ With a few additional p->is_blocked = 0 in a few cases where we return current if blocked_on gets zeroed or there is no owner. This may hint that these current special cases might be dropped eventually. This change also helps resolve wait-queue stalls seen with proxy-execution. See previous patch attempts for details: https://lore.kernel.org/lkml/20260430215103.2978955-2-jstultz@google.com/ Reported-by: Vineeth Pillai <vineethrp@google.com> Suggested-by: Peter Zijlstra <peterz@infradead.org> Signed-off-by: John Stultz <jstultz@google.com> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Link: https://patch.msgid.link/20260512025635.2840817-7-jstultz@google.com
1 parent f13beb0 commit 4c2a204

2 files changed

Lines changed: 20 additions & 3 deletions

File tree

include/linux/sched.h

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -852,7 +852,11 @@ struct task_struct {
852852
struct alloc_tag *alloc_tag;
853853
#endif
854854

855-
int on_cpu;
855+
u8 on_cpu;
856+
u8 on_rq;
857+
u8 is_blocked;
858+
u8 __pad;
859+
856860
struct __call_single_node wake_entry;
857861
unsigned int wakee_flips;
858862
unsigned long wakee_flip_decay_ts;
@@ -867,7 +871,6 @@ struct task_struct {
867871
*/
868872
int recent_used_cpu;
869873
int wake_cpu;
870-
int on_rq;
871874

872875
int prio;
873876
int static_prio;

kernel/sched/core.c

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -624,6 +624,12 @@ int task_llc(const struct task_struct *p)
624624
* [ The astute reader will observe that it is possible for two tasks on one
625625
* CPU to have ->on_cpu = 1 at the same time. ]
626626
*
627+
* p->is_blocked <- { 0, 1 }:
628+
*
629+
* is set by try_to_block_task() and cleared by ttwu_do_wakeup() and tracks
630+
* if the task is blocked. Traditionally this would mirror p->on_rq, however
631+
* due things like DELAY_DEQUEUE and PROXY_EXEC, this can diverge.
632+
*
627633
* task_cpu(p): is changed by set_task_cpu(), the rules are:
628634
*
629635
* - Don't call set_task_cpu() on a blocked task:
@@ -3719,6 +3725,7 @@ ttwu_stat(struct task_struct *p, int cpu, int wake_flags)
37193725
*/
37203726
static inline void ttwu_do_wakeup(struct task_struct *p)
37213727
{
3728+
p->is_blocked = 0;
37223729
WRITE_ONCE(p->__state, TASK_RUNNING);
37233730
trace_sched_wakeup(p);
37243731
}
@@ -4252,6 +4259,7 @@ int try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
42524259
* it disabling IRQs (this allows not taking ->pi_lock).
42534260
*/
42544261
WARN_ON_ONCE(p->se.sched_delayed);
4262+
WARN_ON_ONCE(p->is_blocked);
42554263
/* If p is current, we know we can run here, so clear blocked_on */
42564264
clear_task_blocked_on(p, NULL);
42574265
if (!ttwu_state_match(p, state, &success))
@@ -4563,6 +4571,7 @@ static void __sched_fork(u64 clone_flags, struct task_struct *p)
45634571

45644572
/* A delayed task cannot be in clone(). */
45654573
WARN_ON_ONCE(p->se.sched_delayed);
4574+
WARN_ON_ONCE(p->is_blocked);
45664575

45674576
#ifdef CONFIG_FAIR_GROUP_SCHED
45684577
p->se.cfs_rq = NULL;
@@ -6676,13 +6685,16 @@ static bool try_to_block_task(struct rq *rq, struct task_struct *p,
66766685
unsigned long task_state = *task_state_p;
66776686

66786687
if (signal_pending_state(task_state, p)) {
6688+
p->is_blocked = 0;
66796689
WRITE_ONCE(p->__state, TASK_RUNNING);
66806690
*task_state_p = TASK_RUNNING;
66816691
clear_task_blocked_on(p, NULL);
66826692

66836693
return false;
66846694
}
66856695

6696+
p->is_blocked = 1;
6697+
66866698
/*
66876699
* We check should_block after signal_pending because we
66886700
* will want to wake the task in that case. But if
@@ -6843,6 +6855,7 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf)
68436855
/* if its PROXY_WAKING, do return migration or run if current */
68446856
if (mutex == PROXY_WAKING) {
68456857
if (task_current(rq, p)) {
6858+
p->is_blocked = 0;
68466859
clear_task_blocked_on(p, PROXY_WAKING);
68476860
return p;
68486861
}
@@ -6878,6 +6891,7 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf)
68786891
* just run on this rq), or return-migrate the task.
68796892
*/
68806893
if (task_current(rq, p)) {
6894+
p->is_blocked = 0;
68816895
__clear_task_blocked_on(p, NULL);
68826896
return p;
68836897
}
@@ -7111,7 +7125,7 @@ static void __sched notrace __schedule(int sched_mode)
71117125
clear_task_blocked_on(prev, NULL);
71127126

71137127
rq_set_donor(rq, next);
7114-
if (unlikely(next->blocked_on)) {
7128+
if (unlikely(next->is_blocked && next->blocked_on)) {
71157129
next = find_proxy_task(rq, next, &rf);
71167130
if (!next) {
71177131
zap_balance_callbacks(rq);

0 commit comments

Comments
 (0)