Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit 885ff4d

Browse files
ioworker0ioworker0
committedMar 19, 2025
hung_task: show the blocker task if the task is hung on semaphore
Inspired by mutex blocker tracking[1], this patch makes a trade-off to balance the overhead and utility of the hung task detector. Unlike mutexes, semaphores lack explicit ownership tracking, making it challenging to identify the root cause of hangs. To address this, we introduce a last_holder field to the semaphore structure, which is updated when a task successfully calls down() and cleared during up(). The assumption is that if a task is blocked on a semaphore, the holders must not have released it. While this does not guarantee that the last holder is one of the current blockers, it likely provides a practical hint for diagnosing semaphore-related stalls. With this change, the hung task detector can now show blocker task's info like below: [Thu Mar 13 15:18:38 2025] INFO: task cat:1803 blocked for more than 122 seconds. [Thu Mar 13 15:18:38 2025] Tainted: G OE 6.14.0-rc3+ torvalds#14 [Thu Mar 13 15:18:38 2025] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [Thu Mar 13 15:18:38 2025] task:cat state:D stack:0 pid:1803 tgid:1803 ppid:1057 task_flags:0x400000 flags:0x00000004 [Thu Mar 13 15:18:38 2025] Call trace: [Thu Mar 13 15:18:38 2025] __switch_to+0x1ec/0x380 (T) [Thu Mar 13 15:18:38 2025] __schedule+0xc30/0x44f8 [Thu Mar 13 15:18:38 2025] schedule+0xb8/0x3b0 [Thu Mar 13 15:18:38 2025] schedule_timeout+0x1d0/0x208 [Thu Mar 13 15:18:38 2025] __down_common+0x2d4/0x6f8 [Thu Mar 13 15:18:38 2025] __down+0x24/0x50 [Thu Mar 13 15:18:38 2025] down+0xd0/0x140 [Thu Mar 13 15:18:38 2025] read_dummy+0x3c/0xa0 [hung_task_sem] [Thu Mar 13 15:18:38 2025] full_proxy_read+0xfc/0x1d0 [Thu Mar 13 15:18:38 2025] vfs_read+0x1a0/0x858 [Thu Mar 13 15:18:38 2025] ksys_read+0x100/0x220 [Thu Mar 13 15:18:38 2025] __arm64_sys_read+0x78/0xc8 [Thu Mar 13 15:18:38 2025] invoke_syscall+0xd8/0x278 [Thu Mar 13 15:18:38 2025] el0_svc_common.constprop.0+0xb8/0x298 [Thu Mar 13 15:18:38 2025] do_el0_svc+0x4c/0x88 [Thu Mar 13 15:18:38 2025] el0_svc+0x44/0x108 [Thu Mar 13 15:18:38 2025] el0t_64_sync_handler+0x134/0x160 [Thu Mar 13 15:18:38 2025] el0t_64_sync+0x1b8/0x1c0 [Thu Mar 13 15:18:38 2025] INFO: task cat:1803 blocked on a semaphore likely last held by task cat:1802 [Thu Mar 13 15:18:38 2025] task:cat state:S stack:0 pid:1802 tgid:1802 ppid:1057 task_flags:0x400000 flags:0x00000004 [Thu Mar 13 15:18:38 2025] Call trace: [Thu Mar 13 15:18:38 2025] __switch_to+0x1ec/0x380 (T) [Thu Mar 13 15:18:38 2025] __schedule+0xc30/0x44f8 [Thu Mar 13 15:18:38 2025] schedule+0xb8/0x3b0 [Thu Mar 13 15:18:38 2025] schedule_timeout+0xf4/0x208 [Thu Mar 13 15:18:38 2025] msleep_interruptible+0x70/0x130 [Thu Mar 13 15:18:38 2025] read_dummy+0x48/0xa0 [hung_task_sem] [Thu Mar 13 15:18:38 2025] full_proxy_read+0xfc/0x1d0 [Thu Mar 13 15:18:38 2025] vfs_read+0x1a0/0x858 [Thu Mar 13 15:18:38 2025] ksys_read+0x100/0x220 [Thu Mar 13 15:18:38 2025] __arm64_sys_read+0x78/0xc8 [Thu Mar 13 15:18:38 2025] invoke_syscall+0xd8/0x278 [Thu Mar 13 15:18:38 2025] el0_svc_common.constprop.0+0xb8/0x298 [Thu Mar 13 15:18:38 2025] do_el0_svc+0x4c/0x88 [Thu Mar 13 15:18:38 2025] el0_svc+0x44/0x108 [Thu Mar 13 15:18:38 2025] el0t_64_sync_handler+0x134/0x160 [Thu Mar 13 15:18:38 2025] el0t_64_sync+0x1b8/0x1c0 [1] https://lore.kernel.org/all/174046694331.2194069.15472952050240807469.stgit@mhiramat.tok.corp.google.com Suggested-by: Masami Hiramatsu (Google) <mhiramat@kernel.org> Signed-off-by: Mingzhe Yang <mingzhe.yang@ly.com> Signed-off-by: Lance Yang <ioworker0@gmail.com>
1 parent 6df39a3 commit 885ff4d

File tree

3 files changed

+95
-17
lines changed

3 files changed

+95
-17
lines changed
 

‎include/linux/semaphore.h

+14-1
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,25 @@ struct semaphore {
1616
raw_spinlock_t lock;
1717
unsigned int count;
1818
struct list_head wait_list;
19+
20+
#ifdef CONFIG_DETECT_HUNG_TASK_BLOCKER
21+
unsigned long last_holder;
22+
#endif
1923
};
2024

25+
#ifdef CONFIG_DETECT_HUNG_TASK_BLOCKER
26+
#define __LAST_HOLDER_SEMAPHORE_INITIALIZER \
27+
, .last_holder = 0UL
28+
#else
29+
#define __LAST_HOLDER_SEMAPHORE_INITIALIZER
30+
#endif
31+
2132
#define __SEMAPHORE_INITIALIZER(name, n) \
2233
{ \
2334
.lock = __RAW_SPIN_LOCK_UNLOCKED((name).lock), \
2435
.count = n, \
25-
.wait_list = LIST_HEAD_INIT((name).wait_list), \
36+
.wait_list = LIST_HEAD_INIT((name).wait_list) \
37+
__LAST_HOLDER_SEMAPHORE_INITIALIZER \
2638
}
2739

2840
/*
@@ -47,5 +59,6 @@ extern int __must_check down_killable(struct semaphore *sem);
4759
extern int __must_check down_trylock(struct semaphore *sem);
4860
extern int __must_check down_timeout(struct semaphore *sem, long jiffies);
4961
extern void up(struct semaphore *sem);
62+
extern unsigned long sem_last_holder(struct semaphore *sem);
5063

5164
#endif /* __LINUX_SEMAPHORE_H */

‎kernel/hung_task.c

+35-10
Original file line numberDiff line numberDiff line change
@@ -99,31 +99,56 @@ static struct notifier_block panic_block = {
9999
static void debug_show_blocker(struct task_struct *task)
100100
{
101101
struct task_struct *g, *t;
102-
unsigned long owner, blocker;
102+
unsigned long owner, blocker, blocker_lock_type;
103103

104104
RCU_LOCKDEP_WARN(!rcu_read_lock_held(), "No rcu lock held");
105105

106106
blocker = READ_ONCE(task->blocker);
107-
if (!blocker || !hung_task_blocker_is_type(blocker, BLOCKER_TYPE_MUTEX))
107+
if (!blocker)
108108
return;
109109

110-
owner = mutex_get_owner(
111-
(struct mutex *)hung_task_blocker_to_lock(blocker));
110+
if (hung_task_blocker_is_type(blocker, BLOCKER_TYPE_MUTEX)) {
111+
owner = mutex_get_owner(
112+
(struct mutex *)hung_task_blocker_to_lock(blocker));
113+
blocker_lock_type = BLOCKER_TYPE_MUTEX;
114+
} else if (hung_task_blocker_is_type(blocker, BLOCKER_TYPE_SEM)) {
115+
owner = sem_last_holder(
116+
(struct semaphore *)hung_task_blocker_to_lock(blocker));
117+
blocker_lock_type = BLOCKER_TYPE_SEM;
118+
} else
119+
return;
112120

113121
if (unlikely(!owner)) {
114-
pr_err("INFO: task %s:%d is blocked on a mutex, but the owner is not found.\n",
115-
task->comm, task->pid);
122+
switch (blocker_lock_type) {
123+
case BLOCKER_TYPE_MUTEX:
124+
pr_err("INFO: task %s:%d is blocked on a mutex, but the owner is not found.\n",
125+
task->comm, task->pid);
126+
break;
127+
case BLOCKER_TYPE_SEM:
128+
pr_err("INFO: task %s:%d is blocked on a semaphore, but the last holder is not found.\n",
129+
task->comm, task->pid);
130+
break;
131+
}
116132
return;
117133
}
118134

119135
/* Ensure the owner information is correct. */
120136
for_each_process_thread(g, t) {
121-
if ((unsigned long)t == owner) {
137+
if ((unsigned long)t != owner)
138+
continue;
139+
140+
switch (blocker_lock_type) {
141+
case BLOCKER_TYPE_MUTEX:
122142
pr_err("INFO: task %s:%d is blocked on a mutex likely owned by task %s:%d.\n",
123-
task->comm, task->pid, t->comm, t->pid);
124-
sched_show_task(t);
125-
return;
143+
task->comm, task->pid, t->comm, t->pid);
144+
break;
145+
case BLOCKER_TYPE_SEM:
146+
pr_err("INFO: task %s:%d blocked on a semaphore likely last held by task %s:%d\n",
147+
task->comm, task->pid, t->comm, t->pid);
148+
break;
126149
}
150+
sched_show_task(t);
151+
return;
127152
}
128153
}
129154
#else

‎kernel/locking/semaphore.c

+46-6
Original file line numberDiff line numberDiff line change
@@ -33,12 +33,14 @@
3333
#include <linux/spinlock.h>
3434
#include <linux/ftrace.h>
3535
#include <trace/events/lock.h>
36+
#include <linux/hung_task.h>
3637

3738
static noinline void __down(struct semaphore *sem);
3839
static noinline int __down_interruptible(struct semaphore *sem);
3940
static noinline int __down_killable(struct semaphore *sem);
4041
static noinline int __down_timeout(struct semaphore *sem, long timeout);
4142
static noinline void __up(struct semaphore *sem);
43+
static inline void __sem_acquire(struct semaphore *sem);
4244

4345
/**
4446
* down - acquire the semaphore
@@ -58,7 +60,7 @@ void __sched down(struct semaphore *sem)
5860
might_sleep();
5961
raw_spin_lock_irqsave(&sem->lock, flags);
6062
if (likely(sem->count > 0))
61-
sem->count--;
63+
__sem_acquire(sem);
6264
else
6365
__down(sem);
6466
raw_spin_unlock_irqrestore(&sem->lock, flags);
@@ -82,7 +84,7 @@ int __sched down_interruptible(struct semaphore *sem)
8284
might_sleep();
8385
raw_spin_lock_irqsave(&sem->lock, flags);
8486
if (likely(sem->count > 0))
85-
sem->count--;
87+
__sem_acquire(sem);
8688
else
8789
result = __down_interruptible(sem);
8890
raw_spin_unlock_irqrestore(&sem->lock, flags);
@@ -109,7 +111,7 @@ int __sched down_killable(struct semaphore *sem)
109111
might_sleep();
110112
raw_spin_lock_irqsave(&sem->lock, flags);
111113
if (likely(sem->count > 0))
112-
sem->count--;
114+
__sem_acquire(sem);
113115
else
114116
result = __down_killable(sem);
115117
raw_spin_unlock_irqrestore(&sem->lock, flags);
@@ -139,7 +141,7 @@ int __sched down_trylock(struct semaphore *sem)
139141
raw_spin_lock_irqsave(&sem->lock, flags);
140142
count = sem->count - 1;
141143
if (likely(count >= 0))
142-
sem->count = count;
144+
__sem_acquire(sem);
143145
raw_spin_unlock_irqrestore(&sem->lock, flags);
144146

145147
return (count < 0);
@@ -164,7 +166,7 @@ int __sched down_timeout(struct semaphore *sem, long timeout)
164166
might_sleep();
165167
raw_spin_lock_irqsave(&sem->lock, flags);
166168
if (likely(sem->count > 0))
167-
sem->count--;
169+
__sem_acquire(sem);
168170
else
169171
result = __down_timeout(sem, timeout);
170172
raw_spin_unlock_irqrestore(&sem->lock, flags);
@@ -185,6 +187,12 @@ void __sched up(struct semaphore *sem)
185187
unsigned long flags;
186188

187189
raw_spin_lock_irqsave(&sem->lock, flags);
190+
191+
#ifdef CONFIG_DETECT_HUNG_TASK_BLOCKER
192+
if (READ_ONCE(sem->last_holder) == (unsigned long)current)
193+
WRITE_ONCE(sem->last_holder, 0UL);
194+
#endif
195+
188196
if (likely(list_empty(&sem->wait_list)))
189197
sem->count++;
190198
else
@@ -224,8 +232,12 @@ static inline int __sched ___down_common(struct semaphore *sem, long state,
224232
raw_spin_unlock_irq(&sem->lock);
225233
timeout = schedule_timeout(timeout);
226234
raw_spin_lock_irq(&sem->lock);
227-
if (waiter.up)
235+
if (waiter.up) {
236+
#ifdef CONFIG_DETECT_HUNG_TASK_BLOCKER
237+
WRITE_ONCE(sem->last_holder, (unsigned long)current);
238+
#endif
228239
return 0;
240+
}
229241
}
230242

231243
timed_out:
@@ -242,10 +254,18 @@ static inline int __sched __down_common(struct semaphore *sem, long state,
242254
{
243255
int ret;
244256

257+
#ifdef CONFIG_DETECT_HUNG_TASK_BLOCKER
258+
hung_task_set_blocker(sem, BLOCKER_TYPE_SEM);
259+
#endif
260+
245261
trace_contention_begin(sem, 0);
246262
ret = ___down_common(sem, state, timeout);
247263
trace_contention_end(sem, ret);
248264

265+
#ifdef CONFIG_DETECT_HUNG_TASK_BLOCKER
266+
hung_task_clear_blocker();
267+
#endif
268+
249269
return ret;
250270
}
251271

@@ -277,3 +297,23 @@ static noinline void __sched __up(struct semaphore *sem)
277297
waiter->up = true;
278298
wake_up_process(waiter->task);
279299
}
300+
301+
#ifdef CONFIG_DETECT_HUNG_TASK_BLOCKER
302+
unsigned long sem_last_holder(struct semaphore *sem)
303+
{
304+
return READ_ONCE(sem->last_holder);
305+
}
306+
#else
307+
unsigned long sem_last_holder(struct semaphore *sem)
308+
{
309+
return 0UL;
310+
}
311+
#endif
312+
313+
static inline void __sem_acquire(struct semaphore *sem)
314+
{
315+
sem->count--;
316+
#ifdef CONFIG_DETECT_HUNG_TASK_BLOCKER
317+
WRITE_ONCE(sem->last_holder, (unsigned long)current);
318+
#endif
319+
}

0 commit comments

Comments
 (0)
Please sign in to comment.