Skip to content

Commit 3920059

Browse files
Kenoclaude
authored andcommitted
Switch to the new address space as soon as we see an exec
Between a PTRACE_EVENT_EXEC stop and post_exec(), will_schedule() could write the CPU number to the old image's rseq area, which may now be ld.so text (exec_from_other_thread-32 crashed that way in CI). Call post_exec() in record_step() as soon as the scheduler has collected an exec stop, before will_schedule(). If the stop came for the old thread-group leader, which another thread's exec killed, retire the leader there first, recording its exit if we hadn't handled it, and take the thread that exec'd instead. Tell the two apart by whether the task is in an execve: with an emulated ptracer that hasn't reaped a killed thread yet, the thread group can still have other tasks when the exec stop comes. post_exec() copes with a task that died since its stop. Test: exec_rseq, exec_leader_exit_race, ptrace_exec_multithreaded. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent d3d1d5f commit 3920059

11 files changed

Lines changed: 383 additions & 47 deletions

‎CMakeLists.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1091,6 +1091,7 @@ set(BASIC_TESTS
10911091
exec_from_main_thread
10921092
exec_from_other_thread
10931093
exec_killed_before_exit_event
1094+
exec_rseq
10941095
exec_stopsig
10951096
execveat
10961097
exit_killed_during_preparation
@@ -1314,6 +1315,7 @@ set(BASIC_TESTS
13141315
ptrace_event_stop_sigcont
13151316
ptrace_exec
13161317
x86/ptrace_exec32
1318+
ptrace_exec_multithreaded
13171319
ptrace_group_stop
13181320
ptrace_interrupt_syscall
13191321
ptrace_kill_grandtracee
@@ -1618,6 +1620,7 @@ set(TESTS_WITH_PROGRAM
16181620
exclusion_region
16191621
exec_failed
16201622
exec_killed_before_scratch
1623+
exec_leader_exit_race
16211624
exec_many
16221625
exec_shared_as
16231626
execve_loop

‎src/RecordSession.cc‎

Lines changed: 66 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -761,46 +761,7 @@ bool RecordSession::handle_ptrace_event(RecordTask** t_ptr,
761761
}
762762

763763
case PTRACE_EVENT_EXEC: {
764-
if (t->thread_group()->task_set().size() > 1) {
765-
// All tasks but the task that did the execve should have exited by
766-
// now and notified us of their exits. However, it's possible that
767-
// while running the thread-group leader, our PTRACE_CONT raced with its
768-
// PTRACE_EVENT_EXIT and it exited, and the next event we got is this
769-
// PTRACE_EVENT_EXEC after the exec'ing task changed its tid to the
770-
// leader's tid. Or maybe there are kernel bugs; on
771-
// 4.2.0-42-generic running exec_from_other_thread, we reproducibly
772-
// enter PTRACE_EVENT_EXEC for the thread-group leader without seeing
773-
// its PTRACE_EVENT_EXIT.
774-
775-
// So, record this task's exit and destroy it.
776-
// XXX We can't do record_robust_futex_changes here because the address
777-
// space has already gone. That would only matter if some of them were
778-
// in memory accessible to another process even after exec, i.e. a
779-
// shared-memory mapping or two different thread-groups sharing the same
780-
// address space.
781-
pid_t tid = t->rec_tid;
782-
WaitStatus status = t->status();
783-
t->record_exit_trace_event(WaitStatus(0));
784-
t->record_exit_event();
785-
// Don't call RecordTask::destroy() because we don't want to
786-
// PTRACE_DETACH.
787-
delete t;
788-
// Steal the exec'ing task and make it the thread-group leader, and
789-
// carry on!
790-
t = revive_task_for_exec(tid);
791-
scheduler().set_current(t);
792-
*t_ptr = t;
793-
// Tell t that it is actually stopped, because the stop we got is really
794-
// for this task, not the old dead task.
795-
if (!t->did_waitpid(status)) {
796-
// This is totally untested and almost certainly broken, but if the
797-
// task was SIGKILLed out of the EXEC stop then we should probably
798-
// just pretend the exec never happened.
799-
step_state->continue_type = CONTINUE_SYSCALL;
800-
break;
801-
}
802-
}
803-
t->post_exec();
764+
// record_step() has called switch_to_exec_image() for this stop.
804765
t->session().scheduler().did_exit_execve(t);
805766

806767
// This is purely for testing purposes. See RRCALL_KILL_AFTER_EXEC_STOP.
@@ -1476,8 +1437,7 @@ bool RecordSession::kill_exec_task_for_testing(pid_t rec_tid, int point) {
14761437
// only serializes them later on), so try each of them.
14771438
for (Task* tt : tg->task_set()) {
14781439
RecordTask* rt = static_cast<RecordTask*>(tt);
1479-
if (rt->ev().is_syscall_event() && rt->ev().Syscall().is_exec() &&
1480-
rt->take_armed_kill(point)) {
1440+
if (rt->is_in_execve() && rt->take_armed_kill(point)) {
14811441
syscall(SYS_tgkill, rec_tid, rec_tid, SIGKILL);
14821442
if (point == RRCALL_KILL_BEFORE_EXEC_GETEVENTMSG_AT_EXIT_STOP) {
14831443
WaitOptions options(rec_tid);
@@ -1532,6 +1492,55 @@ RecordTask* RecordSession::revive_task_for_exec(pid_t rec_tid) {
15321492
return t;
15331493
}
15341494

1495+
void RecordSession::dispose_of_leader_killed_by_exec(RecordTask* leader) {
1496+
if (!leader->handled_ptrace_exit_event()) {
1497+
// We haven't handled the leader's PTRACE_EVENT_EXIT. Either it never
1498+
// stopped there, because it was already exiting (e.g. in SYS_exit or
1499+
// exit_group) when the exec killed it, or the exec's SIGKILL took it out
1500+
// of that stop before we handled it. Record its exit now.
1501+
// did_die_in_exec() marks the exit as handled, which ~Task() requires,
1502+
// and the task as reaped, so we don't touch it through its tid, which
1503+
// now refers to the new image; e.g. record_event() leaves the leader's
1504+
// rseq area alone.
1505+
// The ticks hpc.stop() reads are lost, as in handle_ptrace_exit_event();
1506+
// replay doesn't check them for EV_EXIT.
1507+
// XXX We can't do record_robust_futex_changes here because the address
1508+
// space has already gone. That would only matter if some of them were
1509+
// in memory accessible to another process even after exec, i.e. a
1510+
// shared-memory mapping or two different thread-groups sharing the same
1511+
// address space.
1512+
leader->did_die_in_exec();
1513+
leader->hpc.stop(leader);
1514+
leader->record_exit_trace_event(WaitStatus(0));
1515+
leader->record_exit_event();
1516+
}
1517+
// Don't call RecordTask::destroy() because we don't want to PTRACE_DETACH.
1518+
delete leader;
1519+
}
1520+
1521+
RecordTask* RecordSession::switch_to_exec_image(RecordTask* t) {
1522+
if (!t->is_in_execve()) {
1523+
// t isn't in an execve, so another thread of t's thread group did the
1524+
// exec. The exec killed t, the thread-group leader, and the thread that
1525+
// exec'd has taken over t's tid; we collected its exec stop for t, e.g.
1526+
// because we hadn't handled t's exit yet.
1527+
pid_t tid = t->tid;
1528+
WaitStatus status = t->status();
1529+
dispose_of_leader_killed_by_exec(t);
1530+
t = revive_task_for_exec(tid);
1531+
scheduler().set_current(t);
1532+
// The stop we got is really for this task.
1533+
if (!t->did_waitpid(status)) {
1534+
return nullptr;
1535+
}
1536+
}
1537+
if (!t->did_post_exec_at_exec_stop) {
1538+
t->post_exec();
1539+
t->did_post_exec_at_exec_stop = true;
1540+
}
1541+
return t;
1542+
}
1543+
15351544
/**
15361545
* Take a NativeArch::siginfo_t& here instead of siginfo_t because different
15371546
* versions of system headers have inconsistent field naming.
@@ -2778,6 +2787,21 @@ RecordSession::RecordResult RecordSession::record_step() {
27782787
}
27792788
prev_task->pop_event(EV_SCHED);
27802789
}
2790+
// This comes before switch_to_exec_image(): at the initial exec stop,
2791+
// post_exec() sets done_initial_exec(), which ends these kills.
2792+
maybe_kill_task_before_initial_exec(t);
2793+
if (rescheduled.by_waitpid && t->ptrace_event() == PTRACE_EVENT_EXEC) {
2794+
// t's tid now refers to the new image. Switch to it before anything
2795+
// (e.g. will_schedule() writing to the rseq area) uses the old image's
2796+
// state through that tid.
2797+
t = switch_to_exec_image(t);
2798+
if (!t) {
2799+
// The task that did the exec was SIGKILLed out of the exec stop.
2800+
// Pretend this stop never happened and wait for its exit event.
2801+
last_task_switchable = ALLOW_SWITCH;
2802+
return result;
2803+
}
2804+
}
27812805
if (t->tuid() != prev_task_tuid) {
27822806
t->will_schedule();
27832807
}
@@ -2792,7 +2816,6 @@ RecordSession::RecordResult RecordSession::record_step() {
27922816
t->log_pending_events();
27932817
}
27942818

2795-
maybe_kill_task_before_initial_exec(t);
27962819
if (handle_ptrace_exit_event(t)) {
27972820
// t may have been deleted.
27982821
last_task_switchable = ALLOW_SWITCH;

‎src/RecordSession.h‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -222,6 +222,25 @@ class RecordSession final : public Session {
222222
void on_destroy_record_task(RecordTask* t);
223223

224224
private:
225+
/**
226+
* Called when the scheduler returns |t| at a PTRACE_EVENT_EXEC stop it
227+
* collected for |t|, whose tid now refers to the new image. It can return
228+
* the same stop again in a later step if |t| hasn't been resumed since.
229+
* Returns the task that did the exec, switched to the new image with
230+
* post_exec() once per stop. If that task isn't |t|, |t| is the
231+
* thread-group leader, which the exec killed: we record its exit if we
232+
* haven't yet, destroy it, and revive_task_for_exec(). Returns null if
233+
* the task that did the exec was killed before we could read its
234+
* registers.
235+
*/
236+
RecordTask* switch_to_exec_image(RecordTask* t);
237+
238+
/**
239+
* Record the exit of the thread-group leader |leader|, which a non-leader
240+
* thread's exec killed, if we haven't yet, and destroy it.
241+
*/
242+
void dispose_of_leader_killed_by_exec(RecordTask* leader);
243+
225244
RecordSession(const std::string& exe_path,
226245
const std::vector<std::string>& argv,
227246
const std::vector<std::string>& envp,

‎src/RecordTask.cc‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,7 @@ RecordTask::RecordTask(RecordSession& session, pid_t _tid, uint32_t serial,
174174
emulated_ptrace_cont_command(0),
175175
emulated_stop_pending(false),
176176
held_at_emulated_ptrace_event(false),
177+
did_post_exec_at_exec_stop(false),
177178
emulated_ptrace_SIGCHLD_pending(false),
178179
emulated_SIGCHLD_pending(false),
179180
emulated_ptrace_seized(false),
@@ -678,6 +679,7 @@ bool RecordTask::is_at_syscallstub_exit_breakpoint() {
678679
void RecordTask::will_resume_execution(ResumeRequest, WaitRequest,
679680
TicksRequest ticks_request, int sig) {
680681
held_at_emulated_ptrace_event = false;
682+
did_post_exec_at_exec_stop = false;
681683

682684
// We may execute user code, which could lead to an RDTSC or grow-map
683685
// operation which unblocks SIGSEGV, and we'll need to know whether to
@@ -2101,7 +2103,9 @@ void RecordTask::record_event(Event ev, FlushSyscallbuf flush,
21012103
}
21022104
}
21032105

2104-
if (trace_writer().clear_fip_fdp()) {
2106+
// A reaped task is gone; anything we'd read or write through its tid
2107+
// could belong to another task (see Task::did_die_in_exec()).
2108+
if (trace_writer().clear_fip_fdp() && !was_reaped()) {
21052109
const ExtraRegisters* maybe_extra = extra_regs_fallible();
21062110
if (maybe_extra) {
21072111
ExtraRegisters extra_registers = *maybe_extra;

‎src/RecordTask.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -498,6 +498,10 @@ class RecordTask final : public Task {
498498
/** Return the event at the top of this's stack. */
499499
Event& ev() { return pending_events.back(); }
500500
const Event& ev() const { return pending_events.back(); }
501+
/** Return true if this is in an execve or execveat. */
502+
bool is_in_execve() const {
503+
return ev().is_syscall_event() && ev().Syscall().is_exec();
504+
}
501505

502506
/**
503507
* Obtain the previous event on the stack (if any) or nullptr (if not)
@@ -790,6 +794,9 @@ class RecordTask final : public Task {
790794
// (e.g. PTRACE_EVENT_FORK) to the ptracer. prepare_ptrace_cont resumes the
791795
// task when the ptracer does. Cleared whenever the task is resumed.
792796
bool held_at_emulated_ptrace_event;
797+
// true when we've called post_exec() at the PTRACE_EVENT_EXEC stop the
798+
// task is at. Cleared whenever the task is resumed.
799+
bool did_post_exec_at_exec_stop;
793800
// true if this task needs to send a SIGCHLD to its ptracer for its
794801
// emulated ptrace stop
795802
bool emulated_ptrace_SIGCHLD_pending;

‎src/Task.cc‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -490,6 +490,11 @@ void Task::did_kill()
490490
cloned_file_data_fd_child = -1;
491491
}
492492

493+
void Task::did_die_in_exec() {
494+
did_kill();
495+
was_reaped_ = true;
496+
}
497+
493498
/**
494499
* Must be idempotent.
495500
*/
@@ -1107,9 +1112,14 @@ void Task::post_exec(const string& exe_file) {
11071112

11081113
extra_registers = ExtraRegisters(registers.arch());
11091114
extra_registers_known = false;
1110-
ExtraRegisters e = *extra_regs_fallible();
1111-
e.reset();
1112-
set_extra_regs(e);
1115+
// This fails if the task has died (e.g. it was SIGKILLed) since we read
1116+
// its registers.
1117+
const ExtraRegisters* maybe_extra = extra_regs_fallible();
1118+
if (maybe_extra) {
1119+
ExtraRegisters e = *maybe_extra;
1120+
e.reset();
1121+
set_extra_regs(e);
1122+
}
11131123

11141124
syscallbuf_child = nullptr;
11151125
syscallbuf_size = 0;

‎src/Task.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -309,6 +309,14 @@ class Task {
309309

310310
void did_kill();
311311

312+
/**
313+
* Call when another thread's exec has killed this task, the thread-group
314+
* leader, and de_thread() has reaped it before we handled its
315+
* PTRACE_EVENT_EXIT. Like did_kill(), and also marks the task as reaped
316+
* so nothing touches it through its tid.
317+
*/
318+
void did_die_in_exec();
319+
312320
void unmap_buffers_for(
313321
AutoRemoteSyscalls& remote, Task* t,
314322
remote_ptr<struct syscallbuf_hdr> saved_syscallbuf_child);

‎src/test/exec_leader_exit_race.c‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
/* -*- Mode: C; tab-width: 8; c-basic-offset: 2; indent-tabs-mode: nil; -*- */
2+
3+
#include "util.h"
4+
5+
/* A child's main thread creates a thread and exits with SYS_exit; the
6+
thread sleeps briefly and execs. The exec kills the main thread, and the
7+
exec'ing thread takes over its tid. rr can collect the PTRACE_EVENT_EXEC
8+
stop before it has handled the main thread's exit; it must then record
9+
that exit and carry on with the thread that exec'd. The race it tests
10+
only happens in some recordings. */
11+
12+
#define NUM_ROUNDS 40
13+
14+
static char* exe;
15+
16+
static void* do_exec(__attribute__((unused)) void* p) {
17+
char* argv[] = { __atomic_load_n(&exe, __ATOMIC_SEQ_CST), "exec'd", NULL };
18+
usleep(100);
19+
execve(argv[0], argv, environ);
20+
test_assert(0 && "Failed exec!");
21+
return NULL;
22+
}
23+
24+
int main(int argc, char** argv) {
25+
int i;
26+
27+
if (argc > 1) {
28+
test_assert(!strcmp(argv[1], "exec'd"));
29+
return 77;
30+
}
31+
32+
__atomic_store_n(&exe, argv[0], __ATOMIC_SEQ_CST);
33+
for (i = 0; i < NUM_ROUNDS; ++i) {
34+
int status;
35+
pid_t child = fork();
36+
if (!child) {
37+
pthread_t thread;
38+
test_assert(0 == pthread_create(&thread, NULL, do_exec, NULL));
39+
syscall(SYS_exit, 0);
40+
test_assert(0 && "exit returned");
41+
}
42+
test_assert(child > 0);
43+
test_assert(child == waitpid(child, &status, 0));
44+
test_assert(WIFEXITED(status) && WEXITSTATUS(status) == 77);
45+
}
46+
47+
atomic_puts("EXIT-SUCCESS");
48+
return 0;
49+
}

‎src/test/exec_leader_exit_race.run‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
source `dirname $0`/util.sh
2+
3+
RECORD_ARGS="--chaos"
4+
compare_test EXIT-SUCCESS

0 commit comments

Comments
 (0)