Skip to content

Commit 75df60b

Browse files
Kenoclaude
authored andcommitted
Record the SCHED of the previous task before reaping the next one
When a task's timeslice expired and we switched to a task that was only waiting to be reaped, record_step() reaped that task and returned before recording the first task's SCHED. With no current task left, the next step didn't know the previous task, so the SCHED was never recorded, and replay ran what that task executed before the expiry only after the events of the tasks that ran next. Record the SCHED before reaping. The early return came with 3875d7a ("Delay task reaping when necessary, but allow tasks that are ready to reap to be scheduled for reaping"). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent e3619ba commit 75df60b

3 files changed

Lines changed: 152 additions & 8 deletions

File tree

‎CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1372,6 +1372,7 @@ set(BASIC_TESTS
13721372
samask
13731373
save_data_fd
13741374
sched_attr
1375+
sched_reap_zombie
13751376
sched_setaffinity
13761377
sched_setparam
13771378
sched_yield

‎src/RecordSession.cc‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2795,14 +2795,10 @@ RecordSession::RecordResult RecordSession::record_step() {
27952795
// to do something (e.g. terminate the recording).
27962796
return result;
27972797
}
2798-
// If the task has been reaped prematurely then it's not running
2799-
// and we can't get registers etc, so minimize what we do between here
2800-
// to handle_ptrace_exit_event().
2801-
if (t->waiting_for_reap) {
2802-
// Give it another chance to be reaped
2803-
t->did_reach_zombie();
2804-
return result;
2805-
}
2798+
// Record the SCHED of the task we switched away from before the early
2799+
// returns below, so replay runs that task up to where it stopped before
2800+
// what we run next. This includes a switch to a task that we only reap:
2801+
// that leaves no current task, so the next step has no `prev_task`.
28062802
RecordTask* prev_task = find_task(prev_task_tuid);
28072803
if (prev_task && prev_task->ev().type() == EV_SCHED) {
28082804
if (prev_task != t) {
@@ -2812,6 +2808,14 @@ RecordSession::RecordResult RecordSession::record_step() {
28122808
}
28132809
prev_task->pop_event(EV_SCHED);
28142810
}
2811+
// If the task has been reaped prematurely then it's not running
2812+
// and we can't get registers etc, so minimize what we do between here
2813+
// to handle_ptrace_exit_event().
2814+
if (t->waiting_for_reap) {
2815+
// Give it another chance to be reaped
2816+
t->did_reach_zombie();
2817+
return result;
2818+
}
28152819
// This comes before switch_to_exec_image(): at the initial exec stop,
28162820
// post_exec() sets done_initial_exec(), which ends these kills.
28172821
maybe_kill_task_before_initial_exec(t);

‎src/test/sched_reap_zombie.c‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
/* -*- Mode: C; tab-width: 8; c-basic-offset: 2; indent-tabs-mode: nil; -*- */
2+
3+
#include "util.h"
4+
5+
/* A thread-group leader that exits before the group's other threads can't be
6+
* reaped until they are gone too, so rr keeps it to reap later, when the
7+
* scheduler picks it. If the scheduler picks it when another task's timeslice
8+
* expires, rr must still record that task's SCHED event: the tasks that run
9+
* next may run before that task continues, and replay must not run them
10+
* before the part of the task's execution that preceded the expiry.
11+
*
12+
* The main thread spins until the waiter thread sets `done`, which the waiter
13+
* does once waitpid() reports the child's exit, i.e. after rr has reaped the
14+
* child's leader. We arrange for rr to reap the leader when the main thread's
15+
* timeslice expires, and to run the waiter next:
16+
* - The child thread waits until the main thread is blocked, then blocks
17+
* itself, so rr last ran the main thread before it.
18+
* - The child's leader wakes both and exits. rr runs the main thread first.
19+
* - The main thread calls sched_yield(), so rr runs the child thread, which
20+
* exits (now the leader can be reaped), then the main thread again.
21+
* - The main thread spins until its timeslice expires. Then rr picks the
22+
* leader (exiting tasks have the highest priority) and reaps it, which
23+
* lets the waiter's waitpid() return, and runs the waiter, which sets
24+
* `done`.
25+
* Replay must run the main thread's spin up to the expiry before the waiter's
26+
* store to `done`. rr doesn't always schedule as described, so we try a few
27+
* times. */
28+
29+
#define ROUNDS 4
30+
31+
struct waiting {
32+
/* Set to wake the task. */
33+
int go;
34+
/* Set by the task just before it waits for `go`. */
35+
pid_t tid;
36+
};
37+
38+
struct shared {
39+
struct waiting thread;
40+
struct waiting main;
41+
};
42+
43+
struct waiter_state {
44+
pid_t child;
45+
/* Set once waitpid() has reported the child's exit. */
46+
int done;
47+
};
48+
49+
static void wait_for_go(struct waiting* w) {
50+
__atomic_store_n(&w->tid, sys_gettid(), __ATOMIC_SEQ_CST);
51+
while (!__atomic_load_n(&w->go, __ATOMIC_SEQ_CST)) {
52+
syscall(SYS_futex, &w->go, FUTEX_WAIT, 0, NULL, NULL, 0);
53+
}
54+
}
55+
56+
static void go(struct waiting* w) {
57+
__atomic_store_n(&w->go, 1, __ATOMIC_SEQ_CST);
58+
test_assert(syscall(SYS_futex, &w->go, FUTEX_WAKE, 1, NULL, NULL, 0) >= 0);
59+
}
60+
61+
/* Wait until the task that waits with `w` is blocked in its futex wait. We
62+
* observe that while we run, so rr last ran it before us. */
63+
static void wait_until_waiting(struct waiting* w) {
64+
struct timespec ts = { 0, 1000000 };
65+
pid_t tid;
66+
while (!(tid = __atomic_load_n(&w->tid, __ATOMIC_SEQ_CST)) ||
67+
get_task_state(tid) != 'S') {
68+
nanosleep(&ts, NULL);
69+
}
70+
}
71+
72+
static void* waiter(void* p) {
73+
struct waiter_state* state = p;
74+
pid_t child = __atomic_load_n(&state->child, __ATOMIC_SEQ_CST);
75+
int status;
76+
test_assert(child > 1);
77+
test_assert(child == waitpid(child, &status, 0));
78+
test_assert(WIFEXITED(status) && WEXITSTATUS(status) == 0);
79+
__atomic_store_n(&state->done, 1, __ATOMIC_SEQ_CST);
80+
return NULL;
81+
}
82+
83+
static void* child_thread(void* p) {
84+
struct shared* s = p;
85+
wait_until_waiting(&s->main);
86+
wait_for_go(&s->thread);
87+
/* The child's last thread: the child exits with the leader's status 0. */
88+
syscall(SYS_exit, 0);
89+
return NULL;
90+
}
91+
92+
static void do_round(struct shared* s) {
93+
struct waiter_state state;
94+
pthread_t thread;
95+
uint64_t iterations = 0;
96+
pid_t child;
97+
98+
__atomic_store_n(&s->thread.go, 0, __ATOMIC_SEQ_CST);
99+
__atomic_store_n(&s->thread.tid, 0, __ATOMIC_SEQ_CST);
100+
__atomic_store_n(&s->main.go, 0, __ATOMIC_SEQ_CST);
101+
__atomic_store_n(&s->main.tid, 0, __ATOMIC_SEQ_CST);
102+
child = fork();
103+
test_assert(child >= 0);
104+
if (!child) {
105+
pthread_t t;
106+
test_assert(0 == pthread_create(&t, NULL, child_thread, s));
107+
wait_until_waiting(&s->thread);
108+
go(&s->thread);
109+
go(&s->main);
110+
/* The thread-group leader exits first. */
111+
syscall(SYS_exit, 0);
112+
exit(1);
113+
}
114+
115+
__atomic_store_n(&state.child, child, __ATOMIC_SEQ_CST);
116+
__atomic_store_n(&state.done, 0, __ATOMIC_SEQ_CST);
117+
test_assert(0 == pthread_create(&thread, NULL, waiter, &state));
118+
wait_for_go(&s->main);
119+
sched_yield();
120+
while (!__atomic_load_n(&state.done, __ATOMIC_SEQ_CST)) {
121+
++iterations;
122+
}
123+
test_assert(0 == pthread_join(thread, NULL));
124+
atomic_printf("spun %s\n", iterations > 0 ? "a while" : "not at all");
125+
}
126+
127+
int main(void) {
128+
struct shared* s;
129+
int i;
130+
131+
s = (struct shared*)mmap(NULL, sizeof(*s), PROT_READ | PROT_WRITE,
132+
MAP_SHARED | MAP_ANONYMOUS, -1, 0);
133+
test_assert(s != MAP_FAILED);
134+
for (i = 0; i < ROUNDS; ++i) {
135+
do_round(s);
136+
}
137+
atomic_puts("EXIT-SUCCESS");
138+
return 0;
139+
}

0 commit comments

Comments
 (0)