From c0109388684ec7a423913dbcab05c41da22418e0 Mon Sep 17 00:00:00 2001 From: Sean Cheatham Date: Sat, 11 Jul 2026 13:38:09 +0000 Subject: [PATCH 1/2] Add core-dump and boost::stacktrace crash tooling across all xrpld nodes --- CLAUDE.md | 8 ++ Dockerfile.xrpld | 46 ++++++- crash-handler/crash_handler.cpp | 121 ++++++++++++++++++ fuzzer-entrypoint.sh | 17 +++ .../templates/fuzzer_service.yml.mako | 5 + .../templates/service.yml.mako | 7 + 6 files changed, 200 insertions(+), 4 deletions(-) create mode 100644 crash-handler/crash_handler.cpp diff --git a/CLAUDE.md b/CLAUDE.md index 08baaf7..20a85ed 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -130,3 +130,11 @@ Caveat: `sometimes(success)` + `conf_mpt_version_monotonic` only fire against an ### Randomness & specs All randomness via `workload.randoms` (`AntithesisRandom`); generators in `params.py`, never hardcode. Tx docs: `xrpl.org/docs/references/protocol/transactions/types/`; specs: `github.com/XRPLF/XRPL-Standards` `XLS-NNNN-/`. + +## Crash diagnostics (xrpld image) + +`xrpld` is an unstripped Debug build with full DWARF (`/symbols/xrpld`), so crashes are fully symbolizable — but a raw SIGSEGV leaves nothing in `debug.log`. Two mechanisms capture it, both applied at the **image/compose** level so they cover **every** node built on this image (validators, the tracking-node peer, and the fuzzer), not just whatever runs `fuzzer-entrypoint.sh`: +- **boost::stacktrace handler** (`crash-handler/crash_handler.cpp`, built header-only w/ addr2line backend). Installed via `/etc/ld.so.preload` (`Dockerfile.xrpld`) so it loads into every dynamically-linked process regardless of launch path, and composes with any platform `LD_PRELOAD` (e.g. libvoidstar) instead of being overridden by a runtime env. On a fatal signal it `safe_dump_to()`s a raw stack (async-signal-safe), then re-raises to also produce a core; the next start symbolizes the dump to `file:line` on stderr. Symbolization assumes stable load addresses (holds under Antithesis's disabled ASLR); the core is the ASLR-independent fallback. +- **Core dumps** — enabled via the compose `ulimits: core` on every service (`prepare-workload/` `service.yml.mako` + `fuzzer_service.yml.mako`), so validators/peer get cores even though they launch `xrpld` directly with no entrypoint. `fuzzer-entrypoint.sh` additionally best-efforts `core_pattern` → `/var/log/xrpld/cores/` (host-global sysctl, may be read-only). Runtime image ships `gdb`/`binutils`/`elfutils`/`file` to open cores in-container. + +Launch gotcha: validators (`val0`–`valN`) and the tracking-node peer (`xrpld`) run the `xrpld` binary directly via `command:` in the generated compose; only the `fuzzer` container runs `fuzzer-entrypoint.sh` (which also starts an isolated xrpld internally). That's why the two mechanisms live in the image (`ld.so.preload`) and the compose templates (`ulimits`) rather than the entrypoint. An in-process backtrace-on-SIGSEGV still belongs upstream in rippled itself. diff --git a/Dockerfile.xrpld b/Dockerfile.xrpld index 36a8451..b0c9150 100644 --- a/Dockerfile.xrpld +++ b/Dockerfile.xrpld @@ -156,11 +156,35 @@ EOF RUN curl -fsSL https://antithesis.com/assets/instrumentation/libvoidstar.so \ -o /root/libvoidstar.so +# Build the LD_PRELOAD crash handler against the boost headers already resolved +# in the conan cache by the xrpld install above. Header-only (addr2line backend → +# only boost headers + -ldl needed, no boost libs). Static libstdc++/libgcc to +# match rippled (-static-libstdc++) so the preload doesn't drag a second dynamic +# C++ runtime into the process. +COPY crash-handler/crash_handler.cpp /root/crash-handler/crash_handler.cpp +RUN < /etc/ld.so.preload # Copy entrypoint script COPY fuzzer-entrypoint.sh /opt/fuzzer/fuzzer-entrypoint.sh diff --git a/crash-handler/crash_handler.cpp b/crash-handler/crash_handler.cpp new file mode 100644 index 0000000..2dce190 --- /dev/null +++ b/crash-handler/crash_handler.cpp @@ -0,0 +1,121 @@ +// Crash handler for xrpld / rippled-fuzzer, installed process-wide via +// /etc/ld.so.preload (see Dockerfile.xrpld) so it loads into every node built on +// the image — validators and the tracking-node peer launch xrpld directly with no +// entrypoint script, so an LD_PRELOAD env set only in fuzzer-entrypoint.sh would +// miss them. +// +// On a fatal signal (SIGSEGV/SIGABRT/SIGBUS/SIGFPE/SIGILL) it writes an +// async-signal-safe raw stack dump, then restores the default disposition and +// re-raises — so, with core dumps enabled (compose `ulimits: core`), the crash +// ALSO leaves a core. On the next start it symbolizes any leftover dump into +// stderr (which the container captures) as file:line frames. +// +// Why two phases: symbolizing a stack to text allocates and shells out to +// addr2line, neither of which is async-signal-safe. boost::stacktrace's +// safe_dump_to() only records raw return addresses (signal-safe); from_dump() +// does the unsafe symbolization later, in a healthy process. +// +// Symbolization on restart assumes stable module load addresses between the +// crashing and recovering runs. Antithesis runs deterministically with ASLR +// disabled, so the absolute addresses in the dump resolve correctly. The +// re-raised core dump is the ASLR-independent fallback: open it in gdb against +// /symbols/xrpld (the binary ships full DWARF). +// +// Built header-only with the addr2line backend (-DBOOST_STACKTRACE_USE_ADDR2LINE +// + -ldl); see Dockerfile.xrpld. Linked -static-libstdc++/-static-libgcc to +// match rippled and avoid interposing a second dynamic C++ runtime. + +#define _GNU_SOURCE 1 + +#include + +#include // program_invocation_short_name +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace { + +char g_dump_path[4096] = {0}; + +// async-signal-safe write of a NUL-terminated string +void safe_write(int fd, char const* s) +{ + ssize_t n = ::write(fd, s, std::strlen(s)); + (void)n; +} + +extern "C" void crash_handler(int signum) +{ + // Restore the default so the re-raise performs the normal action + // (terminate + core dump) instead of re-entering this handler. + std::signal(signum, SIG_DFL); + + safe_write(STDERR_FILENO, "\n*** fatal signal caught \xe2\x80\x94 dumping stack to "); + safe_write(STDERR_FILENO, g_dump_path); + safe_write(STDERR_FILENO, " ***\n"); + + // Signal-safe: records raw return addresses only, no symbolization. + boost::stacktrace::safe_dump_to(g_dump_path); + + std::raise(signum); +} + +void symbolize_previous_dump() +{ + std::ifstream in(g_dump_path, std::ios::binary); + if (!in.good()) + return; + + // No const char* overload in boost::stacktrace::from_dump — it takes an + // istream (reads the raw frame pointers written by safe_dump_to). + boost::stacktrace::stacktrace const st = + boost::stacktrace::stacktrace::from_dump(in); + in.close(); + + // Write via ::write rather than std::cerr: this runs from the preload's + // constructor, before main(), where the std::cerr global's init ordering + // relative to ours isn't guaranteed. + std::ostringstream oss; + oss << "=== recovered crash backtrace (from " << g_dump_path << ") ===\n" + << st << "=== end crash backtrace ===\n"; + std::string const out = oss.str(); + ssize_t const n = ::write(STDERR_FILENO, out.data(), out.size()); + (void)n; + + // Remove so the same crash isn't re-reported on every later start. + std::remove(g_dump_path); +} + +__attribute__((constructor)) void install() +{ + // Per-process dump path so xrpld and rippled-fuzzer don't clobber each + // other. Overridable with XRPLD_STACKDUMP (a full path). + char const* env = std::getenv("XRPLD_STACKDUMP"); + if (env && *env) + { + std::snprintf(g_dump_path, sizeof(g_dump_path), "%s", env); + } + else + { + std::snprintf( + g_dump_path, + sizeof(g_dump_path), + "/var/log/xrpld/cores/%s.stackdump", + program_invocation_short_name); + } + + // Report (and clear) a dump left by a previous crashed run, if any. + symbolize_previous_dump(); + + for (int const s : {SIGSEGV, SIGABRT, SIGBUS, SIGFPE, SIGILL}) + std::signal(s, &crash_handler); +} + +} // namespace diff --git a/fuzzer-entrypoint.sh b/fuzzer-entrypoint.sh index e81622d..aeec7a5 100644 --- a/fuzzer-entrypoint.sh +++ b/fuzzer-entrypoint.sh @@ -18,6 +18,23 @@ done echo "Loopback addresses configured:" ip addr show lo | grep "inet " +# Route core dumps to the captured log volume so a crash leaves a post-mortem +# artifact that survives the container. Core dumps themselves are enabled +# image-wide via the compose `ulimits: core` (prepare-workload templates), and the +# boost::stacktrace crash handler is installed via /etc/ld.so.preload +# (Dockerfile.xrpld) — so both also cover the validators/peer, which launch xrpld +# directly and never run this script. core_pattern is a host-global (non-namespaced) +# sysctl: writing it needs a writable /proc and privilege we may not have, so this +# is best-effort; otherwise the host's pattern applies (a relative pattern writes +# the core to the process CWD). +CORE_DIR=/var/log/xrpld/cores +mkdir -p "$CORE_DIR" +if echo "$CORE_DIR/core.%e.%p.%t" > /proc/sys/kernel/core_pattern 2>/dev/null; then + echo "core_pattern -> $CORE_DIR/core.%e.%p.%t" +else + echo "warning: could not set core_pattern (host-managed / read-only)" +fi + echo "Starting rippled-fuzzer" /opt/fuzzer/bin/rippled-fuzzer /etc/opt/fuzzer/fuzzer.cfg & FUZZER_PID=$! diff --git a/prepare-workload/prepare_workload/templates/fuzzer_service.yml.mako b/prepare-workload/prepare_workload/templates/fuzzer_service.yml.mako index 1bae8f5..03ec85c 100644 --- a/prepare-workload/prepare_workload/templates/fuzzer_service.yml.mako +++ b/prepare-workload/prepare_workload/templates/fuzzer_service.yml.mako @@ -3,6 +3,11 @@ container_name: ${container_name} hostname: ${hostname} init: true + # Enable core dumps for the fuzzer + its isolated xrpld (matches service.yml.mako). + ulimits: + core: + soft: -1 + hard: -1 entrypoint: ["/bin/bash", "/opt/fuzzer/fuzzer-entrypoint.sh"] environment: - NUM_REAL_PEERS=${num_real_peers} diff --git a/prepare-workload/prepare_workload/templates/service.yml.mako b/prepare-workload/prepare_workload/templates/service.yml.mako index bf7d6e6..62942c3 100644 --- a/prepare-workload/prepare_workload/templates/service.yml.mako +++ b/prepare-workload/prepare_workload/templates/service.yml.mako @@ -3,6 +3,13 @@ container_name: ${container_name} hostname: ${hostname} init: true + # Enable core dumps so a SIGSEGV/SIGABRT leaves a post-mortem core. Validators + # and the tracking-node peer launch xrpld directly (no entrypoint script), so + # the ulimit must be set here rather than in fuzzer-entrypoint.sh. + ulimits: + core: + soft: -1 + hard: -1 % if command: command: ${command} % endif From 40a12d6097d764cdbbb2c900ed5a91c19e84a16a Mon Sep 17 00:00:00 2001 From: Sean Cheatham Date: Thu, 23 Jul 2026 18:49:45 -0400 Subject: [PATCH 2/2] Integrate GDB indexes into xrpld image --- CLAUDE.md | 6 +- Dockerfile.xrpld | 60 +++------ crash-handler/crash_handler.cpp | 121 ------------------ fuzzer-entrypoint.sh | 17 --- .../templates/fuzzer_service.yml.mako | 5 - .../templates/service.yml.mako | 7 - 6 files changed, 20 insertions(+), 196 deletions(-) delete mode 100644 crash-handler/crash_handler.cpp diff --git a/CLAUDE.md b/CLAUDE.md index 20a85ed..1ac6f5a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -133,8 +133,6 @@ All randomness via `workload.randoms` (`AntithesisRandom`); generators in `param ## Crash diagnostics (xrpld image) -`xrpld` is an unstripped Debug build with full DWARF (`/symbols/xrpld`), so crashes are fully symbolizable — but a raw SIGSEGV leaves nothing in `debug.log`. Two mechanisms capture it, both applied at the **image/compose** level so they cover **every** node built on this image (validators, the tracking-node peer, and the fuzzer), not just whatever runs `fuzzer-entrypoint.sh`: -- **boost::stacktrace handler** (`crash-handler/crash_handler.cpp`, built header-only w/ addr2line backend). Installed via `/etc/ld.so.preload` (`Dockerfile.xrpld`) so it loads into every dynamically-linked process regardless of launch path, and composes with any platform `LD_PRELOAD` (e.g. libvoidstar) instead of being overridden by a runtime env. On a fatal signal it `safe_dump_to()`s a raw stack (async-signal-safe), then re-raises to also produce a core; the next start symbolizes the dump to `file:line` on stderr. Symbolization assumes stable load addresses (holds under Antithesis's disabled ASLR); the core is the ASLR-independent fallback. -- **Core dumps** — enabled via the compose `ulimits: core` on every service (`prepare-workload/` `service.yml.mako` + `fuzzer_service.yml.mako`), so validators/peer get cores even though they launch `xrpld` directly with no entrypoint. `fuzzer-entrypoint.sh` additionally best-efforts `core_pattern` → `/var/log/xrpld/cores/` (host-global sysctl, may be read-only). Runtime image ships `gdb`/`binutils`/`elfutils`/`file` to open cores in-container. +`xrpld` is an unstripped Debug build with full DWARF (`/symbols/xrpld`). Debug a crash in an MVD session: rewind to the crash moment and attach gdb to the live replayed process (`thread apply all bt`), or `gcore` it. No on-disk core is needed: the replay is the source of truth, Antithesis captures cores externally, and in-guest `core_pattern` isn't reachable. -Launch gotcha: validators (`val0`–`valN`) and the tracking-node peer (`xrpld`) run the `xrpld` binary directly via `command:` in the generated compose; only the `fuzzer` container runs `fuzzer-entrypoint.sh` (which also starts an isolated xrpld internally). That's why the two mechanisms live in the image (`ld.so.preload`) and the compose templates (`ulimits`) rather than the entrypoint. An in-process backtrace-on-SIGSEGV still belongs upstream in rippled itself. +Gotcha: `gdb-add-index` (`Dockerfile.xrpld`, runtime stage) embeds a `.gdb_index` into the binaries at build time. Without it gdb rebuilds its symbol table by scanning ~600 MB of DWARF at every startup (minutes), overrunning the MVD command window; with it gdb loads in seconds. `binutils` provides the `objcopy` it uses. diff --git a/Dockerfile.xrpld b/Dockerfile.xrpld index b0c9150..eea132e 100644 --- a/Dockerfile.xrpld +++ b/Dockerfile.xrpld @@ -156,35 +156,11 @@ EOF RUN curl -fsSL https://antithesis.com/assets/instrumentation/libvoidstar.so \ -o /root/libvoidstar.so -# Build the LD_PRELOAD crash handler against the boost headers already resolved -# in the conan cache by the xrpld install above. Header-only (addr2line backend → -# only boost headers + -ldl needed, no boost libs). Static libstdc++/libgcc to -# match rippled (-static-libstdc++) so the preload doesn't drag a second dynamic -# C++ runtime into the process. -COPY crash-handler/crash_handler.cpp /root/crash-handler/crash_handler.cpp -RUN < /etc/ld.so.preload +# gdb opens a core / attaches to a live process to produce a symbolized backtrace +# in MVD; binutils supplies objcopy (used by gdb-add-index below) plus objdump/ +# readelf as gdb companions. The xrpld binary ships full DWARF + symtab (unstripped +# Debug build), so no separate symbol package is needed. +RUN set -ex; apt-get update && apt-get install --yes curl jq procps gdb binutils iproute2 && rm -rf /var/lib/apt/lists/* + +# Embed a .gdb_index in the binaries (gdb-add-index ships with gdb; it uses objcopy +# from binutils). Without it, gdb rebuilds its symbol table by scanning ~600 MB of +# DWARF at startup -- significant CPU and memory that, on a resource-constrained MVD +# VM, can overrun the debug-session command window; the index lets gdb jump straight +# to the units it needs. One-time build cost (the scan happens here); the /symbols/* +# links resolve to these same files. +RUN set -ex; \ + gdb-add-index /opt/xrpld/bin/xrpld; \ + gdb-add-index /opt/fuzzer/bin/rippled-fuzzer + +# Create fuzzer config directory and xrpld directories +RUN mkdir -p /etc/fuzzer /var/lib/xrpld/db /var/log/xrpld # Copy entrypoint script COPY fuzzer-entrypoint.sh /opt/fuzzer/fuzzer-entrypoint.sh diff --git a/crash-handler/crash_handler.cpp b/crash-handler/crash_handler.cpp deleted file mode 100644 index 2dce190..0000000 --- a/crash-handler/crash_handler.cpp +++ /dev/null @@ -1,121 +0,0 @@ -// Crash handler for xrpld / rippled-fuzzer, installed process-wide via -// /etc/ld.so.preload (see Dockerfile.xrpld) so it loads into every node built on -// the image — validators and the tracking-node peer launch xrpld directly with no -// entrypoint script, so an LD_PRELOAD env set only in fuzzer-entrypoint.sh would -// miss them. -// -// On a fatal signal (SIGSEGV/SIGABRT/SIGBUS/SIGFPE/SIGILL) it writes an -// async-signal-safe raw stack dump, then restores the default disposition and -// re-raises — so, with core dumps enabled (compose `ulimits: core`), the crash -// ALSO leaves a core. On the next start it symbolizes any leftover dump into -// stderr (which the container captures) as file:line frames. -// -// Why two phases: symbolizing a stack to text allocates and shells out to -// addr2line, neither of which is async-signal-safe. boost::stacktrace's -// safe_dump_to() only records raw return addresses (signal-safe); from_dump() -// does the unsafe symbolization later, in a healthy process. -// -// Symbolization on restart assumes stable module load addresses between the -// crashing and recovering runs. Antithesis runs deterministically with ASLR -// disabled, so the absolute addresses in the dump resolve correctly. The -// re-raised core dump is the ASLR-independent fallback: open it in gdb against -// /symbols/xrpld (the binary ships full DWARF). -// -// Built header-only with the addr2line backend (-DBOOST_STACKTRACE_USE_ADDR2LINE -// + -ldl); see Dockerfile.xrpld. Linked -static-libstdc++/-static-libgcc to -// match rippled and avoid interposing a second dynamic C++ runtime. - -#define _GNU_SOURCE 1 - -#include - -#include // program_invocation_short_name -#include -#include -#include -#include -#include -#include -#include - -#include - -namespace { - -char g_dump_path[4096] = {0}; - -// async-signal-safe write of a NUL-terminated string -void safe_write(int fd, char const* s) -{ - ssize_t n = ::write(fd, s, std::strlen(s)); - (void)n; -} - -extern "C" void crash_handler(int signum) -{ - // Restore the default so the re-raise performs the normal action - // (terminate + core dump) instead of re-entering this handler. - std::signal(signum, SIG_DFL); - - safe_write(STDERR_FILENO, "\n*** fatal signal caught \xe2\x80\x94 dumping stack to "); - safe_write(STDERR_FILENO, g_dump_path); - safe_write(STDERR_FILENO, " ***\n"); - - // Signal-safe: records raw return addresses only, no symbolization. - boost::stacktrace::safe_dump_to(g_dump_path); - - std::raise(signum); -} - -void symbolize_previous_dump() -{ - std::ifstream in(g_dump_path, std::ios::binary); - if (!in.good()) - return; - - // No const char* overload in boost::stacktrace::from_dump — it takes an - // istream (reads the raw frame pointers written by safe_dump_to). - boost::stacktrace::stacktrace const st = - boost::stacktrace::stacktrace::from_dump(in); - in.close(); - - // Write via ::write rather than std::cerr: this runs from the preload's - // constructor, before main(), where the std::cerr global's init ordering - // relative to ours isn't guaranteed. - std::ostringstream oss; - oss << "=== recovered crash backtrace (from " << g_dump_path << ") ===\n" - << st << "=== end crash backtrace ===\n"; - std::string const out = oss.str(); - ssize_t const n = ::write(STDERR_FILENO, out.data(), out.size()); - (void)n; - - // Remove so the same crash isn't re-reported on every later start. - std::remove(g_dump_path); -} - -__attribute__((constructor)) void install() -{ - // Per-process dump path so xrpld and rippled-fuzzer don't clobber each - // other. Overridable with XRPLD_STACKDUMP (a full path). - char const* env = std::getenv("XRPLD_STACKDUMP"); - if (env && *env) - { - std::snprintf(g_dump_path, sizeof(g_dump_path), "%s", env); - } - else - { - std::snprintf( - g_dump_path, - sizeof(g_dump_path), - "/var/log/xrpld/cores/%s.stackdump", - program_invocation_short_name); - } - - // Report (and clear) a dump left by a previous crashed run, if any. - symbolize_previous_dump(); - - for (int const s : {SIGSEGV, SIGABRT, SIGBUS, SIGFPE, SIGILL}) - std::signal(s, &crash_handler); -} - -} // namespace diff --git a/fuzzer-entrypoint.sh b/fuzzer-entrypoint.sh index aeec7a5..e81622d 100644 --- a/fuzzer-entrypoint.sh +++ b/fuzzer-entrypoint.sh @@ -18,23 +18,6 @@ done echo "Loopback addresses configured:" ip addr show lo | grep "inet " -# Route core dumps to the captured log volume so a crash leaves a post-mortem -# artifact that survives the container. Core dumps themselves are enabled -# image-wide via the compose `ulimits: core` (prepare-workload templates), and the -# boost::stacktrace crash handler is installed via /etc/ld.so.preload -# (Dockerfile.xrpld) — so both also cover the validators/peer, which launch xrpld -# directly and never run this script. core_pattern is a host-global (non-namespaced) -# sysctl: writing it needs a writable /proc and privilege we may not have, so this -# is best-effort; otherwise the host's pattern applies (a relative pattern writes -# the core to the process CWD). -CORE_DIR=/var/log/xrpld/cores -mkdir -p "$CORE_DIR" -if echo "$CORE_DIR/core.%e.%p.%t" > /proc/sys/kernel/core_pattern 2>/dev/null; then - echo "core_pattern -> $CORE_DIR/core.%e.%p.%t" -else - echo "warning: could not set core_pattern (host-managed / read-only)" -fi - echo "Starting rippled-fuzzer" /opt/fuzzer/bin/rippled-fuzzer /etc/opt/fuzzer/fuzzer.cfg & FUZZER_PID=$! diff --git a/prepare-workload/prepare_workload/templates/fuzzer_service.yml.mako b/prepare-workload/prepare_workload/templates/fuzzer_service.yml.mako index 03ec85c..1bae8f5 100644 --- a/prepare-workload/prepare_workload/templates/fuzzer_service.yml.mako +++ b/prepare-workload/prepare_workload/templates/fuzzer_service.yml.mako @@ -3,11 +3,6 @@ container_name: ${container_name} hostname: ${hostname} init: true - # Enable core dumps for the fuzzer + its isolated xrpld (matches service.yml.mako). - ulimits: - core: - soft: -1 - hard: -1 entrypoint: ["/bin/bash", "/opt/fuzzer/fuzzer-entrypoint.sh"] environment: - NUM_REAL_PEERS=${num_real_peers} diff --git a/prepare-workload/prepare_workload/templates/service.yml.mako b/prepare-workload/prepare_workload/templates/service.yml.mako index 62942c3..bf7d6e6 100644 --- a/prepare-workload/prepare_workload/templates/service.yml.mako +++ b/prepare-workload/prepare_workload/templates/service.yml.mako @@ -3,13 +3,6 @@ container_name: ${container_name} hostname: ${hostname} init: true - # Enable core dumps so a SIGSEGV/SIGABRT leaves a post-mortem core. Validators - # and the tracking-node peer launch xrpld directly (no entrypoint script), so - # the ulimit must be set here rather than in fuzzer-entrypoint.sh. - ulimits: - core: - soft: -1 - hard: -1 % if command: command: ${command} % endif