Skip to content

Commit 571f4dc

Browse files
Kenoclaude
authored andcommitted
Track fork advice from madvise calls that fail with ENOMEM
madvise() applies the advice to the mapped parts of the range even when it fails with ENOMEM because other parts of the range are unmapped. rr ignored such calls, so a forked child's memory in replay could differ from the recording: e.g. after such a MADV_DONTFORK, the child had the mapped parts in replay but not in the recording. Apply MADV_DONTFORK, MADV_DOFORK, MADV_WIPEONFORK and MADV_KEEPONFORK to the mapped parts of the range, also when madvise fails with ENOMEM. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent faa0a52 commit 571f4dc

5 files changed

Lines changed: 141 additions & 3 deletions

File tree

‎CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1193,6 +1193,7 @@ set(BASIC_TESTS
11931193
lsm_self_attr
11941194
madvise
11951195
madvise_dontneed_private
1196+
madvise_fork_enomem
11961197
madvise_free
11971198
madvise_misc
11981199
madvise_remove

‎src/AddressSpace.cc‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1458,6 +1458,31 @@ void AddressSpace::unmap_internal(Task* t, remote_ptr<void> addr,
14581458
update_watchpoint_values(addr, addr + num_bytes);
14591459
}
14601460

1461+
// madvise applies the advice only to mapped memory, failing with ENOMEM
1462+
// where the range has gaps. Add each contiguous mapped part of |range| as
1463+
// one range.
1464+
void AddressSpace::add_range_where_mapped(set<MemoryRange>& ranges,
1465+
const MemoryRange& range) {
1466+
remote_ptr<void> start;
1467+
remote_ptr<void> end;
1468+
for (const auto& m : maps_containing_or_after(range.start())) {
1469+
MemoryRange r = m.map.intersect(range);
1470+
if (!r.size()) {
1471+
break;
1472+
}
1473+
if (r.start() != end) {
1474+
if (start != end) {
1475+
add_range(ranges, MemoryRange(start, end));
1476+
}
1477+
start = r.start();
1478+
}
1479+
end = r.end();
1480+
}
1481+
if (start != end) {
1482+
add_range(ranges, MemoryRange(start, end));
1483+
}
1484+
}
1485+
14611486
void AddressSpace::advise(Task*, remote_ptr<void> addr, ssize_t num_bytes,
14621487
int advice) {
14631488
LOG(debug) << "madvise(" << addr << ", " << num_bytes << ", " << advice
@@ -1466,13 +1491,13 @@ void AddressSpace::advise(Task*, remote_ptr<void> addr, ssize_t num_bytes,
14661491

14671492
switch (advice) {
14681493
case MADV_DONTFORK:
1469-
add_range(dont_fork, MemoryRange(addr, num_bytes));
1494+
add_range_where_mapped(dont_fork, MemoryRange(addr, num_bytes));
14701495
break;
14711496
case MADV_DOFORK:
14721497
remove_range(dont_fork, MemoryRange(addr, num_bytes));
14731498
break;
14741499
case MADV_WIPEONFORK:
1475-
add_range(wipe_on_fork, MemoryRange(addr, num_bytes));
1500+
add_range_where_mapped(wipe_on_fork, MemoryRange(addr, num_bytes));
14761501
break;
14771502
case MADV_KEEPONFORK:
14781503
remove_range(wipe_on_fork, MemoryRange(addr, num_bytes));

‎src/AddressSpace.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -961,6 +961,8 @@ class AddressSpace : public HasTaskSet {
961961
void populate_address_space(Task* t);
962962

963963
void unmap_internal(Task* t, remote_ptr<void> addr, ssize_t num_bytes);
964+
void add_range_where_mapped(std::set<MemoryRange>& ranges,
965+
const MemoryRange& range);
964966
void update_syscall_ips(Task* t);
965967

966968
bool update_watchpoint_value(const MemoryRange& range,

‎src/Task.cc‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -633,11 +633,15 @@ void Task::on_syscall_exit_arch(int syscallno, const Registers& regs) {
633633

634634
// mprotect can change the protection status of some mapped regions before
635635
// failing.
636+
// madvise applies the advice to the mapped parts of the range when it
637+
// fails with ENOMEM because other parts are unmapped.
636638
// SYS_rrcall_mprotect_record always fails with ENOSYS, though we want to
637639
// note its usage here.
638640
if (regs.syscall_failed() && !is_mprotect_syscall(syscallno, regs.arch())
639641
&& !is_pkey_mprotect_syscall(syscallno, regs.arch())
640-
&& !is_prctl_syscall(syscallno, regs.arch())) {
642+
&& !is_prctl_syscall(syscallno, regs.arch())
643+
&& !(is_madvise_syscall(syscallno, regs.arch()) &&
644+
regs.syscall_result_signed() == -ENOMEM)) {
641645
return;
642646
}
643647

‎src/test/madvise_fork_enomem.c‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
/* -*- Mode: C; tab-width: 8; c-basic-offset: 2; indent-tabs-mode: nil; -*- */
2+
3+
#include "util.h"
4+
#include <setjmp.h>
5+
6+
#ifndef MADV_WIPEONFORK
7+
#define MADV_WIPEONFORK 18
8+
#endif
9+
10+
#ifndef MADV_KEEPONFORK
11+
#define MADV_KEEPONFORK 19
12+
#endif
13+
14+
/* madvise() over a range with an unmapped page applies the advice to the
15+
mapped pages and fails with ENOMEM. The forked child must see the
16+
effects of such calls, in recording and replay. */
17+
18+
static size_t page_size;
19+
static sigjmp_buf jmpbuf;
20+
21+
static void segv_handler(__attribute__((unused)) int sig) {
22+
siglongjmp(jmpbuf, 1);
23+
}
24+
25+
static int is_mapped(volatile char* p) {
26+
if (sigsetjmp(jmpbuf, 1)) {
27+
return 0;
28+
}
29+
(void)*p;
30+
return 1;
31+
}
32+
33+
/* Three pages with the middle one unmapped. */
34+
static char* map_with_hole(void) {
35+
char* p = mmap(NULL, 3 * page_size, PROT_READ | PROT_WRITE,
36+
MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
37+
test_assert(p != MAP_FAILED);
38+
test_assert(0 == munmap(p + page_size, page_size));
39+
p[0] = 1;
40+
p[2 * page_size] = 1;
41+
return p;
42+
}
43+
44+
static void advise_with_hole(char* p, int advice) {
45+
test_assert(-1 == madvise(p, 3 * page_size, advice));
46+
test_assert(ENOMEM == errno);
47+
}
48+
49+
int main(void) {
50+
char* dontfork;
51+
char* dofork;
52+
char* wipe;
53+
char* keep;
54+
int have_wipeonfork;
55+
pid_t pid;
56+
int status;
57+
int i;
58+
59+
page_size = sysconf(_SC_PAGESIZE);
60+
61+
dontfork = map_with_hole();
62+
advise_with_hole(dontfork, MADV_DONTFORK);
63+
64+
dofork = map_with_hole();
65+
test_assert(0 == madvise(dofork, page_size, MADV_DONTFORK));
66+
test_assert(0 == madvise(dofork + 2 * page_size, page_size, MADV_DONTFORK));
67+
advise_with_hole(dofork, MADV_DOFORK);
68+
69+
wipe = map_with_hole();
70+
keep = map_with_hole();
71+
have_wipeonfork = 0 == madvise(keep, page_size, MADV_WIPEONFORK);
72+
if (have_wipeonfork) {
73+
test_assert(0 ==
74+
madvise(keep + 2 * page_size, page_size, MADV_WIPEONFORK));
75+
advise_with_hole(keep, MADV_KEEPONFORK);
76+
advise_with_hole(wipe, MADV_WIPEONFORK);
77+
} else {
78+
atomic_puts("MADV_WIPEONFORK not supported, skipping that part.");
79+
}
80+
81+
pid = fork();
82+
test_assert(pid >= 0);
83+
if (!pid) {
84+
struct sigaction sa;
85+
memset(&sa, 0, sizeof(sa));
86+
sa.sa_handler = segv_handler;
87+
test_assert(0 == sigaction(SIGSEGV, &sa, NULL));
88+
89+
for (i = 0; i < 3; i += 2) {
90+
test_assert(!is_mapped(dontfork + i * page_size));
91+
test_assert(dofork[i * page_size] == 1);
92+
if (have_wipeonfork) {
93+
test_assert(wipe[i * page_size] == 0);
94+
test_assert(keep[i * page_size] == 1);
95+
}
96+
}
97+
98+
atomic_puts("EXIT-SUCCESS");
99+
return 77;
100+
}
101+
102+
test_assert(pid == wait(&status));
103+
test_assert(WIFEXITED(status) && WEXITSTATUS(status) == 77);
104+
105+
return 0;
106+
}

0 commit comments

Comments
 (0)