From 362c2a10f1c5f66dfdddf46c1afac51f1041bb94 Mon Sep 17 00:00:00 2001 From: Stephen Chen <20940639+stephenctw@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:02:09 +0800 Subject: [PATCH 1/5] fix(security): hide secrets in --help, add -32012 retry code, fix watchdog LUA_CPATH - Add `hide_env_values = true` to all private-key and key-file CLI args (KeyArgs + OptionalKeyArgs) so `--help` prints the env var name but never the value. Also hide BLOCKCHAIN_HTTP_ENDPOINT (may contain API tokens in the URL). Closes F1 from the security audit. - Add -32012 to DEFAULT_LONG_BLOCK_RANGE_ERROR_CODES in the sequencer (Rust) and watchdog (Lua), plus the shared test fixture. This code is returned by JSON-RPC proxies when getLogs exceeds max allowed range; the partition retry now bisects on it automatically. - Fix watchdog entrypoint (sequencer-watchdog): append the cartesi deb Lua paths as fallbacks instead of prepending, so the container's /opt/watchdog/lib/lcurl.so is found first. - Add regression tests: help output must not leak secret env values, -32012 must be in default codes and trigger retry, fixture scenario for proxy range-exceeded bisect. --- sequencer/src/l1/partition.rs | 27 +++++++++- sequencer/src/runtime/config.rs | 70 +++++++++++++++++++++++-- tests/fixtures/l1_partition_vector.json | 16 +++++- watchdog/config.lua | 2 +- watchdog/l1_reader.lua | 1 + watchdog/sequencer-watchdog | 15 +++--- 6 files changed, 115 insertions(+), 16 deletions(-) diff --git a/sequencer/src/l1/partition.rs b/sequencer/src/l1/partition.rs index 1f80708..69fa071 100644 --- a/sequencer/src/l1/partition.rs +++ b/sequencer/src/l1/partition.rs @@ -8,8 +8,10 @@ //! This module is stateless: callers pass the retry error codes explicitly. There is no //! global mutable state; `RunConfig` owns the codes and passes them down via configs. -/// Default RPC error codes that trigger partition retry (e.g. Infura -32005, Alchemy -32600/-32602, QuickNode -32616). -pub const DEFAULT_LONG_BLOCK_RANGE_ERROR_CODES: &[&str] = &["-32005", "-32600", "-32602", "-32616"]; +/// Default RPC error codes that trigger partition retry (e.g. Infura -32005, +/// Alchemy -32600/-32602, QuickNode -32616, generic JSON-RPC proxies -32012). +pub const DEFAULT_LONG_BLOCK_RANGE_ERROR_CODES: &[&str] = + &["-32005", "-32012", "-32600", "-32602", "-32616"]; use alloy::contract::Error as ContractError; use alloy::contract::Event; @@ -373,6 +375,27 @@ mod tests { assert!(!error_message_matches_retry_codes("ok", &[])); } + #[test] + fn default_codes_include_proxy_range_exceeded() { + assert!( + DEFAULT_LONG_BLOCK_RANGE_ERROR_CODES.contains(&"-32012"), + "default codes must include -32012 (JSON-RPC proxy range limit)" + ); + } + + #[test] + fn proxy_range_exceeded_error_triggers_retry() { + let msg = r#"-32012: getLogs request exceeded max allowed range, data: {"code":"ErrGetLogsExceededMaxAllowedRange"}"#; + let codes: Vec = DEFAULT_LONG_BLOCK_RANGE_ERROR_CODES + .iter() + .map(|c| (*c).to_string()) + .collect(); + assert!( + error_message_matches_retry_codes(msg, &codes), + "proxy -32012 error must match retry codes" + ); + } + #[test] fn decode_evm_advance_input_round_trips() { let encoded = EvmAdvanceCall { diff --git a/sequencer/src/runtime/config.rs b/sequencer/src/runtime/config.rs index bc4cea7..28d6573 100644 --- a/sequencer/src/runtime/config.rs +++ b/sequencer/src/runtime/config.rs @@ -116,6 +116,7 @@ pub struct KeyArgs { #[arg( long, env = "CARTESI_SEQUENCER_AUTH_PRIVATE_KEY", + hide_env_values = true, group = "batch_submitter_key_source" )] batch_submitter_private_key: Option, @@ -123,6 +124,7 @@ pub struct KeyArgs { #[arg( long, env = "CARTESI_SEQUENCER_AUTH_PRIVATE_KEY_FILE", + hide_env_values = true, group = "batch_submitter_key_source" )] batch_submitter_private_key_file: Option, @@ -167,10 +169,18 @@ fn resolve_key_source( #[derive(Debug, Clone, Args)] pub struct OptionalKeyArgs { /// Hex-encoded batch-submitter private key. - #[arg(long, env = "CARTESI_SEQUENCER_AUTH_PRIVATE_KEY")] + #[arg( + long, + env = "CARTESI_SEQUENCER_AUTH_PRIVATE_KEY", + hide_env_values = true + )] batch_submitter_private_key: Option, /// Path to a file whose first line is the batch-submitter private key. - #[arg(long, env = "CARTESI_SEQUENCER_AUTH_PRIVATE_KEY_FILE")] + #[arg( + long, + env = "CARTESI_SEQUENCER_AUTH_PRIVATE_KEY_FILE", + hide_env_values = true + )] batch_submitter_private_key_file: Option, } @@ -204,7 +214,7 @@ impl OptionalKeyArgs { pub struct SetupConfig { #[arg(long, env = "CARTESI_SEQUENCER_DATA_DIR", default_value = DEFAULT_DATA_DIR, value_parser = parse_non_empty_string)] pub data_dir: String, - #[arg(long, env = "CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT", value_parser = parse_non_empty_string)] + #[arg(long, env = "CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT", hide_env_values = true, value_parser = parse_non_empty_string)] pub eth_rpc_url: String, /// Allow plaintext (`http://`) RPC to a non-loopback host — a trusted /// private network (Docker/K8s service name, `host.docker.internal`, a @@ -330,7 +340,7 @@ pub struct RunConfig { pub http_addr: String, #[arg(long, env = "CARTESI_SEQUENCER_DATA_DIR", default_value = DEFAULT_DATA_DIR, value_parser = parse_non_empty_string)] pub data_dir: String, - #[arg(long, env = "CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT", value_parser = parse_non_empty_string)] + #[arg(long, env = "CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT", hide_env_values = true, value_parser = parse_non_empty_string)] pub eth_rpc_url: String, /// Allow plaintext (`http://`) RPC to a non-loopback host — a trusted /// private network (Docker/K8s service name, `host.docker.internal`, a @@ -411,7 +421,7 @@ impl RunConfig { pub struct FlushConfig { #[arg(long, env = "CARTESI_SEQUENCER_DATA_DIR", default_value = DEFAULT_DATA_DIR, value_parser = parse_non_empty_string)] pub data_dir: String, - #[arg(long, env = "CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT", value_parser = parse_non_empty_string)] + #[arg(long, env = "CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT", hide_env_values = true, value_parser = parse_non_empty_string)] pub eth_rpc_url: String, /// Allow plaintext (`http://`) RPC to a non-loopback host — a trusted /// private network (Docker/K8s service name, `host.docker.internal`, a @@ -664,6 +674,7 @@ mod tests { config.long_block_range_error_codes, vec![ "-32005".to_string(), + "-32012".to_string(), "-32600".to_string(), "-32602".to_string(), "-32616".to_string() @@ -727,6 +738,55 @@ mod tests { ); } + /// Render the `--help` text for a subcommand and return it as a string. + fn render_help(args: &[&str]) -> String { + let err = Cli::try_parse_from(args).unwrap_err(); + err.to_string() + } + + #[test] + fn help_does_not_leak_private_key_value() { + let sentinel_key = "0xSENTINEL_PRIVATE_KEY_DEADBEEF1234567890abcdef"; + let sentinel_rpc = "https://eth-mainnet.g.alchemy.com/v2/SECRET_TOKEN_XYZ"; + + unsafe { + std::env::set_var("CARTESI_SEQUENCER_AUTH_PRIVATE_KEY", sentinel_key); + std::env::set_var( + "CARTESI_SEQUENCER_AUTH_PRIVATE_KEY_FILE", + "/secret/path/key.pem", + ); + std::env::set_var("CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT", sentinel_rpc); + } + + for subcmd in ["run", "setup", "flush-mempool"] { + let help = render_help(&["sequencer", subcmd, "--help"]); + + assert!( + !help.contains(sentinel_key), + "{subcmd} --help must not print the private key value" + ); + assert!( + !help.contains("/secret/path/key.pem"), + "{subcmd} --help must not print the key-file path" + ); + assert!( + !help.contains("SECRET_TOKEN_XYZ"), + "{subcmd} --help must not print the RPC URL (may contain API tokens)" + ); + + assert!( + help.contains("CARTESI_SEQUENCER_AUTH_PRIVATE_KEY"), + "{subcmd} --help should still mention the env var name" + ); + } + + unsafe { + std::env::remove_var("CARTESI_SEQUENCER_AUTH_PRIVATE_KEY"); + std::env::remove_var("CARTESI_SEQUENCER_AUTH_PRIVATE_KEY_FILE"); + std::env::remove_var("CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT"); + } + } + #[test] fn domain_uses_fixed_name_and_version() { use alloy_primitives::U256; diff --git a/tests/fixtures/l1_partition_vector.json b/tests/fixtures/l1_partition_vector.json index 18bdf3f..bec9121 100644 --- a/tests/fixtures/l1_partition_vector.json +++ b/tests/fixtures/l1_partition_vector.json @@ -1,5 +1,5 @@ { - "long_block_range_error_codes": ["-32005", "-32600", "-32602", "-32616"], + "long_block_range_error_codes": ["-32005", "-32012", "-32600", "-32602", "-32616"], "scenarios": [ { "name": "bisect_on_default_infura_code", @@ -65,6 +65,20 @@ ], "expect_ok": true }, + { + "name": "bisect_on_proxy_range_exceeded_code", + "start_block": 100, + "end_block": 200, + "fail_ranges": [ + { + "from": 100, + "to": 200, + "message": "-32012: getLogs request exceeded max allowed range" + } + ], + "expect_calls": [[100, 200], [100, 150], [151, 200]], + "expect_ok": true + }, { "name": "single_block_fail_no_split", "start_block": 7, diff --git a/watchdog/config.lua b/watchdog/config.lua index 4dc5972..9107050 100644 --- a/watchdog/config.lua +++ b/watchdog/config.lua @@ -86,7 +86,7 @@ function config.load_init(env) retry_attempts = optional_number("CARTESI_WATCHDOG_RETRY_ATTEMPTS", 3, env), retry_delay_sec = optional_number("CARTESI_WATCHDOG_RETRY_DELAY_SEC", 5, env), long_block_range_error_codes = split_csv( - env.CARTESI_WATCHDOG_LONG_BLOCK_RANGE_ERROR_CODES or "-32005,-32600,-32602,-32616" + env.CARTESI_WATCHDOG_LONG_BLOCK_RANGE_ERROR_CODES or "-32005,-32012,-32600,-32602,-32616" ), } end diff --git a/watchdog/l1_reader.lua b/watchdog/l1_reader.lua index e796f93..28f5cee 100644 --- a/watchdog/l1_reader.lua +++ b/watchdog/l1_reader.lua @@ -9,6 +9,7 @@ l1_reader.INPUT_ADDED_TOPIC = "0xc05d337121a6e8605c6ec0b72aa29c4210ffe6e5b9cefdd l1_reader.DEFAULT_LONG_BLOCK_RANGE_ERROR_CODES = { "-32005", + "-32012", "-32600", "-32602", "-32616", diff --git a/watchdog/sequencer-watchdog b/watchdog/sequencer-watchdog index 19df961..1fb5dc6 100755 --- a/watchdog/sequencer-watchdog +++ b/watchdog/sequencer-watchdog @@ -11,15 +11,16 @@ fi lua_root="${CARTESI_WATCHDOG_LUA_ROOT:-/opt/watchdog/lua}" lua_bin="${CARTESI_WATCHDOG_LUA_BIN:-lua5.4}" -# Prefer deb-installed cartesi bindings under /usr/lib over stale /usr/local -# copies left by an older manual install (same ordering as the cartesi-machine -# wrapper shipped in machine-emulator_*.deb). +# Append deb-installed cartesi bindings under /usr/lib as a fallback — they +# must come *after* the container's own paths (set via Dockerfile ENV) so that +# /opt/watchdog/lib/lcurl.so and other explicitly placed .so files are found +# first. Prepending broke lcurl discovery in published container images. _cartesi_lua_cpath="/usr/lib/lua/5.4/?.so;/usr/lib/lua/5.4/?/init.so" _cartesi_lua_path="/usr/share/lua/5.4/?.lua;/usr/share/lua/5.4/?/init.lua" -export LUA_CPATH="${_cartesi_lua_cpath};${LUA_CPATH:-;}" -export LUA_PATH="${_cartesi_lua_path};${LUA_PATH:-;}" -export LUA_CPATH_5_4="${_cartesi_lua_cpath};${LUA_CPATH_5_4:-;}" -export LUA_PATH_5_4="${_cartesi_lua_path};${LUA_PATH_5_4:-;}" +export LUA_CPATH="${LUA_CPATH:-;}${_cartesi_lua_cpath};;" +export LUA_PATH="${LUA_PATH:-;}${_cartesi_lua_path};;" +export LUA_CPATH_5_4="${LUA_CPATH_5_4:-;}${_cartesi_lua_cpath};;" +export LUA_PATH_5_4="${LUA_PATH_5_4:-;}${_cartesi_lua_path};;" cd "${lua_root}" From daa2d010f035b400a185b3f31cdf4675798475cf Mon Sep 17 00:00:00 2001 From: Stephen Chen <20940639+stephenctw@users.noreply.github.com> Date: Thu, 30 Jul 2026 18:22:41 +0800 Subject: [PATCH 2/5] fix: address PR review on secrets help test, LUA_CPATH join, docs - Document the conscious accept of bare -32012 in the partition default list (same overload model as existing -32005; no message heuristics). - Rewrite help-leak regression as a child-process probe via Command::env so parallel cargo test no longer mutates process-global env. - Join watchdog LUA_{C,}PATH appends with an explicit ';' and derive the base from versioned-or-unversioned (Lua 5.4). - Update README default for LONG_BLOCK_RANGE_ERROR_CODES to include -32012. --- README.md | 2 +- sequencer/src/l1/partition.rs | 14 +++++- sequencer/src/runtime/config.rs | 80 ++++++++++++++++++++++++--------- watchdog/l1_reader.lua | 4 ++ watchdog/sequencer-watchdog | 38 ++++++++++++---- 5 files changed, 106 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 71da28b..5367d6a 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,7 @@ nonce on demand (keyed operator tool). `setup` requires: `CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT`, `CARTESI_SEQUENCER_BLOCKCHAIN_ID`, `CARTESI_SEQUENCER_APP_ADDRESS`, `CARTESI_SEQUENCER_BATCH_SUBMITTER_ADDRESS`. `run` requires: `CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT`, `CARTESI_SEQUENCER_AUTH_PRIVATE_KEY` (or `_FILE`); it refuses to boot until `setup` has completed. -Optional: `CARTESI_SEQUENCER_HTTP_ADDR` (default `127.0.0.1:3000`, `run`), `CARTESI_SEQUENCER_DATA_DIR` (default `sequencer-data` — SQLite file is `sequencer.db` inside; created if missing), `CARTESI_SEQUENCER_PREEMPTIVE_MARGIN_BLOCKS` (default `300`), `CARTESI_SEQUENCER_SECONDS_PER_BLOCK` (default `12`), `CARTESI_SEQUENCER_L1_READ_STALE_AFTER_BLOCKS` (default `600`), `CARTESI_SEQUENCER_LONG_BLOCK_RANGE_ERROR_CODES` (default `-32005,-32600,-32602,-32616`), `CARTESI_SEQUENCER_AUTH_PRIVATE_KEY_FILE` (alternative to `CARTESI_SEQUENCER_AUTH_PRIVATE_KEY`; first line of the file is the key), `CARTESI_SEQUENCER_BATCH_SUBMITTER_IDLE_POLL_INTERVAL_MS`, `CARTESI_SEQUENCER_BATCH_SUBMITTER_CONFIRMATION_DEPTH`. +Optional: `CARTESI_SEQUENCER_HTTP_ADDR` (default `127.0.0.1:3000`, `run`), `CARTESI_SEQUENCER_DATA_DIR` (default `sequencer-data` — SQLite file is `sequencer.db` inside; created if missing), `CARTESI_SEQUENCER_PREEMPTIVE_MARGIN_BLOCKS` (default `300`), `CARTESI_SEQUENCER_SECONDS_PER_BLOCK` (default `12`), `CARTESI_SEQUENCER_L1_READ_STALE_AFTER_BLOCKS` (default `600`), `CARTESI_SEQUENCER_LONG_BLOCK_RANGE_ERROR_CODES` (default `-32005,-32012,-32600,-32602,-32616`), `CARTESI_SEQUENCER_AUTH_PRIVATE_KEY_FILE` (alternative to `CARTESI_SEQUENCER_AUTH_PRIVATE_KEY`; first line of the file is the key), `CARTESI_SEQUENCER_BATCH_SUBMITTER_IDLE_POLL_INTERVAL_MS`, `CARTESI_SEQUENCER_BATCH_SUBMITTER_CONFIRMATION_DEPTH`. By default the blockchain endpoint must be `https://` unless its host is loopback (`localhost`, `127.0.0.0/8`, `::1`) — a guard against accidentally sending L1 traffic to a public RPC in the clear. Set `CARTESI_SEQUENCER_ALLOW_INSECURE_RPC=true` (or `--allow-insecure-rpc`) to permit plaintext `http://` to a non-loopback host on a **trusted private network** — e.g. a Docker Compose / Kubernetes service name (`http://anvil:8545`), `host.docker.internal`, or a private-VPC IP. diff --git a/sequencer/src/l1/partition.rs b/sequencer/src/l1/partition.rs index 69fa071..04cf365 100644 --- a/sequencer/src/l1/partition.rs +++ b/sequencer/src/l1/partition.rs @@ -9,7 +9,19 @@ //! global mutable state; `RunConfig` owns the codes and passes them down via configs. /// Default RPC error codes that trigger partition retry (e.g. Infura -32005, -/// Alchemy -32600/-32602, QuickNode -32616, generic JSON-RPC proxies -32012). +/// Alchemy -32600/-32602, QuickNode -32616, Goldsky / some JSON-RPC proxies +/// -32012). +/// +/// **Overloaded codes (accepted).** `-32005` and `-32012` are not range-only +/// across providers: Alloy's transport also treats bare `-32005` as Infura +/// rate-limit, and `-32012` + `"credits"` as QuickNode credit rate-limit. +/// We still match on the bare code — the same policy already used for +/// `-32005` — because range-error messages are not stable across proxies +/// (Goldsky vs custom gateways). A persistent QuickNode credits limit that +/// survives Alloy's own retries can therefore fan an N-block scan into +/// `2N−1` partition queries before failing; operators on QuickNode can drop +/// `-32012` from `CARTESI_SEQUENCER_LONG_BLOCK_RANGE_ERROR_CODES` if that +/// matters. We deliberately do **not** add message-marker heuristics here. pub const DEFAULT_LONG_BLOCK_RANGE_ERROR_CODES: &[&str] = &["-32005", "-32012", "-32600", "-32602", "-32616"]; diff --git a/sequencer/src/runtime/config.rs b/sequencer/src/runtime/config.rs index 28d6573..bd012a8 100644 --- a/sequencer/src/runtime/config.rs +++ b/sequencer/src/runtime/config.rs @@ -738,53 +738,89 @@ mod tests { ); } - /// Render the `--help` text for a subcommand and return it as a string. - fn render_help(args: &[&str]) -> String { - let err = Cli::try_parse_from(args).unwrap_err(); - err.to_string() + /// Render `--help` for `subcmd` inside a child process whose env carries + /// the given secret values. Mutating this process's env is unsound under + /// parallel `cargo test` (and neighbouring Clap parses race on the same + /// vars), so the probe runs via `Command::env` on a re-exec of this + /// test binary. + fn help_text_with_secret_env(subcmd: &str, secrets: &[(&str, &str)]) -> String { + let out_path = std::env::temp_dir().join(format!( + "sequencer-help-leak-{}-{}-{}.txt", + subcmd, + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock") + .as_nanos() + )); + + let exe = std::env::current_exe().expect("current_exe"); + let mut cmd = std::process::Command::new(&exe); + cmd.args([ + "--exact", + "runtime::config::tests::help_does_not_leak_private_key_value", + "--quiet", + ]) + .env("SEQUENCER_HELP_LEAK_OUT", &out_path) + .env("SEQUENCER_HELP_LEAK_SUBCMD", subcmd); + for (key, value) in secrets { + cmd.env(key, value); + } + + let status = cmd.status().expect("spawn help-leak child"); + assert!( + status.success(), + "help-leak child failed for subcommand {subcmd}: {status}" + ); + let help = std::fs::read_to_string(&out_path) + .unwrap_or_else(|err| panic!("read help output {}: {err}", out_path.display())); + let _ = std::fs::remove_file(&out_path); + help } #[test] fn help_does_not_leak_private_key_value() { + // Child arm: render help under the caller's env and write it out. + // Exits the process so we never nest another spawn. + if let Ok(out_path) = std::env::var("SEQUENCER_HELP_LEAK_OUT") { + let subcmd = std::env::var("SEQUENCER_HELP_LEAK_SUBCMD") + .expect("SEQUENCER_HELP_LEAK_SUBCMD must be set in the child"); + let help = Cli::try_parse_from(["sequencer", subcmd.as_str(), "--help"]) + .expect_err("--help must short-circuit as a clap error") + .to_string(); + std::fs::write(&out_path, help).expect("write help output"); + std::process::exit(0); + } + let sentinel_key = "0xSENTINEL_PRIVATE_KEY_DEADBEEF1234567890abcdef"; + let sentinel_file = "/secret/path/key.pem"; let sentinel_rpc = "https://eth-mainnet.g.alchemy.com/v2/SECRET_TOKEN_XYZ"; - - unsafe { - std::env::set_var("CARTESI_SEQUENCER_AUTH_PRIVATE_KEY", sentinel_key); - std::env::set_var( - "CARTESI_SEQUENCER_AUTH_PRIVATE_KEY_FILE", - "/secret/path/key.pem", - ); - std::env::set_var("CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT", sentinel_rpc); - } + let secrets = [ + ("CARTESI_SEQUENCER_AUTH_PRIVATE_KEY", sentinel_key), + ("CARTESI_SEQUENCER_AUTH_PRIVATE_KEY_FILE", sentinel_file), + ("CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT", sentinel_rpc), + ]; for subcmd in ["run", "setup", "flush-mempool"] { - let help = render_help(&["sequencer", subcmd, "--help"]); + let help = help_text_with_secret_env(subcmd, &secrets); assert!( !help.contains(sentinel_key), "{subcmd} --help must not print the private key value" ); assert!( - !help.contains("/secret/path/key.pem"), + !help.contains(sentinel_file), "{subcmd} --help must not print the key-file path" ); assert!( !help.contains("SECRET_TOKEN_XYZ"), "{subcmd} --help must not print the RPC URL (may contain API tokens)" ); - assert!( help.contains("CARTESI_SEQUENCER_AUTH_PRIVATE_KEY"), "{subcmd} --help should still mention the env var name" ); } - - unsafe { - std::env::remove_var("CARTESI_SEQUENCER_AUTH_PRIVATE_KEY"); - std::env::remove_var("CARTESI_SEQUENCER_AUTH_PRIVATE_KEY_FILE"); - std::env::remove_var("CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT"); - } } #[test] diff --git a/watchdog/l1_reader.lua b/watchdog/l1_reader.lua index 28f5cee..d204fd2 100644 --- a/watchdog/l1_reader.lua +++ b/watchdog/l1_reader.lua @@ -7,6 +7,10 @@ local l1_reader = {} l1_reader.INPUT_ADDED_TOPIC = "0xc05d337121a6e8605c6ec0b72aa29c4210ffe6e5b9cefdd6a7058188a8f66f98" +-- Bare codes, matching the sequencer's DEFAULT_LONG_BLOCK_RANGE_ERROR_CODES. +-- `-32005` / `-32012` are provider-overloaded (rate-limit vs range); we accept +-- that the same way the Rust side does — see the comment on that constant. +-- Operators can override via CARTESI_WATCHDOG_LONG_BLOCK_RANGE_ERROR_CODES. l1_reader.DEFAULT_LONG_BLOCK_RANGE_ERROR_CODES = { "-32005", "-32012", diff --git a/watchdog/sequencer-watchdog b/watchdog/sequencer-watchdog index 1fb5dc6..088457b 100755 --- a/watchdog/sequencer-watchdog +++ b/watchdog/sequencer-watchdog @@ -11,16 +11,38 @@ fi lua_root="${CARTESI_WATCHDOG_LUA_ROOT:-/opt/watchdog/lua}" lua_bin="${CARTESI_WATCHDOG_LUA_BIN:-lua5.4}" -# Append deb-installed cartesi bindings under /usr/lib as a fallback — they -# must come *after* the container's own paths (set via Dockerfile ENV) so that -# /opt/watchdog/lib/lcurl.so and other explicitly placed .so files are found -# first. Prepending broke lcurl discovery in published container images. +# Deb-installed cartesi bindings under /usr/lib — appended *after* the +# container's own paths (Dockerfile ENV / operator override) so +# /opt/watchdog/lib/lcurl.so is found first. Prepending broke lcurl +# discovery in published images. Lua 5.4 prefers the versioned +# LUA_{C,}PATH_5_4 variables when set, so derive the effective base from +# versioned-or-unversioned before composing. _cartesi_lua_cpath="/usr/lib/lua/5.4/?.so;/usr/lib/lua/5.4/?/init.so" _cartesi_lua_path="/usr/share/lua/5.4/?.lua;/usr/share/lua/5.4/?/init.lua" -export LUA_CPATH="${LUA_CPATH:-;}${_cartesi_lua_cpath};;" -export LUA_PATH="${LUA_PATH:-;}${_cartesi_lua_path};;" -export LUA_CPATH_5_4="${LUA_CPATH_5_4:-;}${_cartesi_lua_cpath};;" -export LUA_PATH_5_4="${LUA_PATH_5_4:-;}${_cartesi_lua_path};;" + +# Strip trailing semicolons from $1, then append $2 with an explicit `;`, +# and finish with `;;` so Lua still expands its defaults at the end. +join_lua_search_path() { + local base="$1" + local extra="$2" + while [[ "$base" == *'; ]]; do + base="${base%;}" + done + if [[ -z "$base" ]]; then + printf '%s;;' "$extra" + else + printf '%s;%s;;' "$base" "$extra" + fi +} + +_cpath_base="${LUA_CPATH_5_4:-${LUA_CPATH:-}}" +_path_base="${LUA_PATH_5_4:-${LUA_PATH:-}}" +_new_cpath="$(join_lua_search_path "${_cpath_base}" "${_cartesi_lua_cpath}")" +_new_path="$(join_lua_search_path "${_path_base}" "${_cartesi_lua_path}")" +export LUA_CPATH="${_new_cpath}" +export LUA_PATH="${_new_path}" +export LUA_CPATH_5_4="${_new_cpath}" +export LUA_PATH_5_4="${_new_path}" cd "${lua_root}" From 4317e2694496759173a133f6ef78d4290f5804bb Mon Sep 17 00:00:00 2001 From: Stephen Chen <20940639+stephenctw@users.noreply.github.com> Date: Thu, 30 Jul 2026 19:26:39 +0800 Subject: [PATCH 3/5] fix(watchdog): parse-safe LUA_PATH join (bash -n clean) The trailing-semicolon trim used `[[ == *' ; ]]` with a broken quote, so bash treated later `printf '%s;;'` as inside the conditional and aborted every sequencer-watchdog init with unexpected token `;;`. Trim via `case` instead and keep the explicit `;` join. --- watchdog/sequencer-watchdog | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/watchdog/sequencer-watchdog b/watchdog/sequencer-watchdog index 088457b..f2d3c84 100755 --- a/watchdog/sequencer-watchdog +++ b/watchdog/sequencer-watchdog @@ -22,13 +22,18 @@ _cartesi_lua_path="/usr/share/lua/5.4/?.lua;/usr/share/lua/5.4/?/init.lua" # Strip trailing semicolons from $1, then append $2 with an explicit `;`, # and finish with `;;` so Lua still expands its defaults at the end. +# Use `case` (not `[[ == *... ]]`) to trim - a semicolon inside `[[ ]]` +# is a reserved `;;` token and is easy to mis-quote. join_lua_search_path() { local base="$1" local extra="$2" - while [[ "$base" == *'; ]]; do - base="${base%;}" + while true; do + case "$base" in + *";") base="${base%;}" ;; + *) break ;; + esac done - if [[ -z "$base" ]]; then + if [ -z "$base" ]; then printf '%s;;' "$extra" else printf '%s;%s;;' "$base" "$extra" From be510edc27af1c0c0ad0ca530be9175362b96e2a Mon Sep 17 00:00:00 2001 From: Stephen Chen <20940639+stephenctw@users.noreply.github.com> Date: Fri, 31 Jul 2026 09:32:05 +0800 Subject: [PATCH 4/5] fix(watchdog): make init idempotent like sequencer setup Re-running init on an already-initialized state dir is a no-op success (exit 0) so process supervisors can invoke it unconditionally. --- docs/watchdog/README.md | 2 +- docs/watchdog/operator-deployment.md | 11 +++++++---- watchdog/main.lua | 16 +++++++++++++++- watchdog/tests/run.lua | 9 ++++++--- 4 files changed, 29 insertions(+), 9 deletions(-) diff --git a/docs/watchdog/README.md b/docs/watchdog/README.md index cad139d..5809e5c 100644 --- a/docs/watchdog/README.md +++ b/docs/watchdog/README.md @@ -182,7 +182,7 @@ the chain id explicitly). The watchdog has two subcommands: ```bash -sequencer-watchdog init # one-time setup: writes config.json + head.json +sequencer-watchdog init # setup: writes config.json + head.json (idempotent) sequencer-watchdog tick # one compare cycle; schedule this ``` diff --git a/docs/watchdog/operator-deployment.md b/docs/watchdog/operator-deployment.md index fe92da0..805dad2 100644 --- a/docs/watchdog/operator-deployment.md +++ b/docs/watchdog/operator-deployment.md @@ -174,10 +174,13 @@ Pick one: 3. **Replay from genesis** (only for new rollups / low block height — slow). Run `init` once to store the bootstrap CM snapshot into the watchdog state -layout. The L1 RPC URL is not persisted — each `tick` reads -`CARTESI_WATCHDOG_BLOCKCHAIN_HTTP_ENDPOINT` so it can rotate without editing -state. If `CARTESI_WATCHDOG_BLOCKCHAIN_ID` is unset at `init`, auto-detect also -needs that endpoint present then (prefer setting the chain id explicitly): +layout. Re-running `init` on an already-initialized state directory is a no-op +success (exit `0`), matching `sequencer setup` — safe for process supervisors +that always invoke init before tick. The L1 RPC URL is not persisted — each +`tick` reads `CARTESI_WATCHDOG_BLOCKCHAIN_HTTP_ENDPOINT` so it can rotate +without editing state. If `CARTESI_WATCHDOG_BLOCKCHAIN_ID` is unset at `init`, +auto-detect also needs that endpoint present then (prefer setting the chain id +explicitly): ```bash sequencer-watchdog init diff --git a/watchdog/main.lua b/watchdog/main.lua index 89c6a82..76b2f5e 100644 --- a/watchdog/main.lua +++ b/watchdog/main.lua @@ -112,7 +112,14 @@ local function run_init(cfg, deps) local existing, load_err = checkpoint.load(cfg.state_dir) if existing then - return nil, "watchdog state already initialized" + -- Idempotent like `sequencer setup`: re-init on an already-set-up + -- state dir is a no-op success so process supervisors can run init + -- unconditionally without wrapping exit codes. + return { + ok = true, + already_initialized = true, + safe_block = existing.safe_block, + } end if load_err ~= "missing " .. checkpoint.HEAD_FILE then return nil, "failed to load watchdog head: " .. tostring(load_err) @@ -251,6 +258,13 @@ local function main(argv, opts) io.stderr:write("watchdog init failed: " .. tostring(err) .. "\n") os.exit(EXIT_TRANSIENT) end + if result.already_initialized then + io.stderr:write( + "watchdog init already complete — nothing to do state_dir=" + .. tostring(cfg.state_dir) + .. "\n" + ) + end os.exit(EXIT_OK) end diff --git a/watchdog/tests/run.lua b/watchdog/tests/run.lua index 8145373..206db9f 100644 --- a/watchdog/tests/run.lua +++ b/watchdog/tests/run.lua @@ -958,7 +958,7 @@ test("tick config requires current RPC URL outside persisted state", function() assert(tostring(load_err):find("CARTESI_WATCHDOG_BLOCKCHAIN_HTTP_ENDPOINT", 1, true) ~= nil, tostring(load_err)) end) -test("init refuses an already initialized state directory", function() +test("init is a no-op success when state is already initialized", function() local dir = os.tmpname() os.remove(dir) @@ -967,10 +967,13 @@ test("init refuses an already initialized state directory", function() local first, first_err = main_mod.run_init(cfg, { machine = fake_machine("{}") }) assert(first, first_err) + assert_eq(first.already_initialized, nil) local second, second_err = main_mod.run_init(cfg, { machine = fake_machine("{}") }) - assert_eq(second, nil) - assert(tostring(second_err):find("already initialized", 1, true) ~= nil, tostring(second_err)) + assert(second, second_err) + assert_eq(second.ok, true) + assert_eq(second.already_initialized, true) + assert_eq(second.safe_block, first.safe_block) end) test("runner happy path replays inputs and writes checkpoint", function() From 1b36f7b090b82b43ad13f6edcf71de65621bc8bd Mon Sep 17 00:00:00 2001 From: Stephen Chen <20940639+stephenctw@users.noreply.github.com> Date: Fri, 31 Jul 2026 18:26:27 +0800 Subject: [PATCH 5/5] fix: validate complete watchdog state on idempotent init; add permissive CORS Idempotent init now requires readable config.json and a non-empty checkpoint snapshot before exit 0, so supervisors are not told success on unusable state. Document LONG_BLOCK_RANGE_ERROR_CODES as init-only. Enable CorsLayer::permissive for browser POST /tx. --- README.md | 1 + docs/watchdog/README.md | 2 +- docs/watchdog/operator-deployment.md | 17 ++++++---- sequencer/Cargo.toml | 2 +- sequencer/src/http.rs | 6 +++- sequencer/tests/snapshot_endpoints.rs | 30 +++++++++++++++++ watchdog/config.lua | 15 +++++++++ watchdog/main.lua | 47 +++++++++++++++++++++++++-- watchdog/tests/run.lua | 42 ++++++++++++++++++++++++ 9 files changed, 149 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 5367d6a..959152f 100644 --- a/README.md +++ b/README.md @@ -146,6 +146,7 @@ Notes: - payload size is bounded at ingress; oversized requests are rejected before entering the hot path. - overload is enforced at queue admission: if the inclusion-lane queue is full, `POST /tx` returns HTTP `429` with code `OVERLOADED` and message `queue full`. - queue capacity is an internal runtime constant tuned alongside inclusion-lane chunking to absorb short bursts; if this starts triggering persistently, it is a signal to revisit runtime sizing or throughput rather than add another admission layer. +- CORS is currently **permissive** (`Access-Control-Allow-Origin: *`, all methods/headers) so browser wallets can call `POST /tx`. Tighten once the planned ingress/egress port split lands. ### `GET /ws/subscribe?from_offset=` diff --git a/docs/watchdog/README.md b/docs/watchdog/README.md index 5809e5c..82cf077 100644 --- a/docs/watchdog/README.md +++ b/docs/watchdog/README.md @@ -182,7 +182,7 @@ the chain id explicitly). The watchdog has two subcommands: ```bash -sequencer-watchdog init # setup: writes config.json + head.json (idempotent) +sequencer-watchdog init # setup: writes config.json + head.json (idempotent if complete) sequencer-watchdog tick # one compare cycle; schedule this ``` diff --git a/docs/watchdog/operator-deployment.md b/docs/watchdog/operator-deployment.md index 805dad2..d75e1e5 100644 --- a/docs/watchdog/operator-deployment.md +++ b/docs/watchdog/operator-deployment.md @@ -160,6 +160,7 @@ Today `WalletApp::default()` / `WalletConfig::sepolia()` align with Sepolia stag | `CARTESI_WATCHDOG_BLOCKCHAIN_ID` | Chain id label for `status.prom` metrics (prefer set at `init`; optional auto-detect via `eth_chainId` when L1 endpoint is present at `init`) | | `CARTESI_WATCHDOG_METRICS_FILE` | Override path for the Prometheus textfile written by each `tick` | | `CARTESI_WATCHDOG_LUA_DEPS` | `.deps/lua` | +| `CARTESI_WATCHDOG_LONG_BLOCK_RANGE_ERROR_CODES` | Optional CSV of RPC error codes that trigger `eth_getLogs` partition retry. **Evaluated only at `init` and persisted in `config.json`** — not a tick-time override; re-running idempotent `init` does not refresh it. Wipe state and re-init (or edit `config.json`) to change. Default matches the sequencer: `-32005,-32012,-32600,-32602,-32616`. | The sequencer discovers and pins `input_box_address` at startup; use the same values as `CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT` / `CARTESI_SEQUENCER_APP_ADDRESS` configuration. @@ -174,13 +175,15 @@ Pick one: 3. **Replay from genesis** (only for new rollups / low block height — slow). Run `init` once to store the bootstrap CM snapshot into the watchdog state -layout. Re-running `init` on an already-initialized state directory is a no-op -success (exit `0`), matching `sequencer setup` — safe for process supervisors -that always invoke init before tick. The L1 RPC URL is not persisted — each -`tick` reads `CARTESI_WATCHDOG_BLOCKCHAIN_HTTP_ENDPOINT` so it can rotate -without editing state. If `CARTESI_WATCHDOG_BLOCKCHAIN_ID` is unset at `init`, -auto-detect also needs that endpoint present then (prefer setting the chain id -explicitly): +layout. Re-running `init` on a **complete** already-initialized state directory +is a no-op success (exit `0`), matching `sequencer setup` — safe for process +supervisors that always invoke init before tick. If `head.json` exists but +`config.json` or the selected snapshot is missing/corrupt, `init` fails (exit +`1`) and asks you to wipe `state_dir` and re-run — it will not certify an +unusable state. The L1 RPC URL is not persisted — each `tick` reads +`CARTESI_WATCHDOG_BLOCKCHAIN_HTTP_ENDPOINT` so it can rotate without editing +state. If `CARTESI_WATCHDOG_BLOCKCHAIN_ID` is unset at `init`, auto-detect also +needs that endpoint present then (prefer setting the chain id explicitly): ```bash sequencer-watchdog init diff --git a/sequencer/Cargo.toml b/sequencer/Cargo.toml index be99c5b..41c61f1 100644 --- a/sequencer/Cargo.toml +++ b/sequencer/Cargo.toml @@ -18,7 +18,7 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" toml = "0.8" tracing = "0.1" -tower-http = { version = "0.6.8", features = ["trace"] } +tower-http = { version = "0.6.8", features = ["trace", "cors"] } rusqlite = { version = "0.38.0", features = ["bundled"] } rusqlite_migration = "2.3.0" alloy-primitives = { version = "1.4.1", features = ["serde", "k256"] } diff --git a/sequencer/src/http.rs b/sequencer/src/http.rs index 7aa4127..264a9ed 100644 --- a/sequencer/src/http.rs +++ b/sequencer/src/http.rs @@ -21,6 +21,7 @@ use axum::response::{IntoResponse, Response}; use serde::Serialize; use thiserror::Error; use tokio::sync::mpsc; +use tower_http::cors::CorsLayer; use tower_http::trace::TraceLayer; pub use crate::egress::api::SnapshotState; @@ -228,7 +229,10 @@ pub fn start_on_listener( )) // Enforces a raw request-body cap before JSON deserialization, including whitespace. .layer(DefaultBodyLimit::max(config.max_body_bytes)) - .layer(TraceLayer::new_for_http()); + .layer(TraceLayer::new_for_http()) + // Permissive CORS so browser wallets can POST /tx (and preflight OPTIONS). + // Tighten when the ingress/egress port split lands and public exposure is narrower. + .layer(CorsLayer::permissive()); tokio::spawn(async move { axum::serve(listener, app) diff --git a/sequencer/tests/snapshot_endpoints.rs b/sequencer/tests/snapshot_endpoints.rs index 840541d..a3e96ae 100644 --- a/sequencer/tests/snapshot_endpoints.rs +++ b/sequencer/tests/snapshot_endpoints.rs @@ -522,3 +522,33 @@ async fn latest_snapshot_falls_back_to_finalized_when_no_pending() { assert!(wait_for_lease(db.path.as_str(), fin_id, 0).await); } + +#[tokio::test] +async fn cors_permits_browser_preflight_on_tx() { + let db = temp_db("cors-preflight"); + let Some(server) = start_server(db.path.as_str()).await else { + return; + }; + + let resp = reqwest::Client::new() + .request(reqwest::Method::OPTIONS, server.url("/tx")) + .header("Origin", "https://wallet.example") + .header("Access-Control-Request-Method", "POST") + .header("Access-Control-Request-Headers", "content-type") + .send() + .await + .expect("OPTIONS /tx"); + + assert!( + resp.status().is_success(), + "preflight status: {}", + resp.status() + ); + let allow_origin = resp + .headers() + .get("access-control-allow-origin") + .expect("Access-Control-Allow-Origin") + .to_str() + .expect("header utf8"); + assert_eq!(allow_origin, "*"); +} diff --git a/watchdog/config.lua b/watchdog/config.lua index 9107050..fca9f3f 100644 --- a/watchdog/config.lua +++ b/watchdog/config.lua @@ -125,6 +125,21 @@ function config.persisted(cfg) } end +--- Validate a persisted config.json object (no tick-time env required). +--- Raises on missing/invalid fields; used by idempotent init before exit 0. +function config.validate_persisted(data) + if type(data) ~= "table" then + error("config.json is not an object") + end + if data.version ~= config.VERSION then + error("unsupported config.json version: " .. tostring(data.version)) + end + required_field(data, "sequencer_url") + required_field(data, "input_box_address") + required_field(data, "app_address") + return true +end + function config.from_persisted(state_dir, data, env) env = normalize_env(env) if data.version ~= config.VERSION then diff --git a/watchdog/main.lua b/watchdog/main.lua index 76b2f5e..1c3b795 100644 --- a/watchdog/main.lua +++ b/watchdog/main.lua @@ -106,15 +106,56 @@ local function default_deps(cfg) return deps, json end +local function shell_quote(value) + value = tostring(value) + return "'" .. value:gsub("'", "'\\''") .. "'" +end + +--- True when `path` is a non-empty directory (CM snapshot must have content). +local function directory_nonempty(path) + local quoted = shell_quote(path) + local ok = os.execute("test -d " .. quoted .. " && test -n \"$(ls -A " .. quoted .. " 2>/dev/null)\"") + return ok == true or ok == 0 +end + +--- Require a complete, tick-usable state before reporting idempotent init success. +--- A valid head.json alone is not enough — config.json or the selected snapshot +--- may be missing/corrupt after a partial wipe. +local function validate_initialized_state(state_dir, existing, json) + local persisted, cfg_err = state.read_json(state_dir, "config.json", json) + if not persisted then + return nil, + "incomplete watchdog state: missing or unreadable config.json (" + .. tostring(cfg_err) + .. ")" + end + local ok, validate_err = pcall(config.validate_persisted, persisted) + if not ok then + return nil, "incomplete watchdog state: " .. tostring(validate_err) + end + if type(existing.snapshot_dir) ~= "string" or existing.snapshot_dir == "" then + return nil, "incomplete watchdog state: head has no snapshot_dir" + end + if not directory_nonempty(existing.snapshot_dir) then + return nil, + "incomplete watchdog state: missing or empty snapshot at " + .. tostring(existing.snapshot_dir) + end + return true +end + local function run_init(cfg, deps) deps = deps or default_machine_deps(cfg) local json = json_mod.new() local existing, load_err = checkpoint.load(cfg.state_dir) if existing then - -- Idempotent like `sequencer setup`: re-init on an already-set-up - -- state dir is a no-op success so process supervisors can run init - -- unconditionally without wrapping exit codes. + local usable, why = validate_initialized_state(cfg.state_dir, existing, json) + if not usable then + return nil, tostring(why) .. "; wipe state_dir and re-run init" + end + -- Idempotent like `sequencer setup`: complete state → no-op success so + -- process supervisors can run init unconditionally. return { ok = true, already_initialized = true, diff --git a/watchdog/tests/run.lua b/watchdog/tests/run.lua index 206db9f..cada214 100644 --- a/watchdog/tests/run.lua +++ b/watchdog/tests/run.lua @@ -521,6 +521,13 @@ local function fake_machine(inspect_state) function machine:dump(_instance, snapshot_dir, reference_block) self.saved_snapshot_dir = snapshot_dir _instance.reference_block = reference_block + -- Real CM dumps create a non-empty snapshot dir; mirror that so + -- idempotent-init usability checks see a complete state. + os.execute("mkdir -p " .. "'" .. tostring(snapshot_dir):gsub("'", "'\\''") .. "'") + local marker = io.open(snapshot_dir .. "/.dump", "w") + assert(marker, "write snapshot marker") + marker:write("ok") + marker:close() return true end function machine:feed_inputs(instance, inputs) @@ -976,6 +983,41 @@ test("init is a no-op success when state is already initialized", function() assert_eq(second.safe_block, first.safe_block) end) +test("init fails when config.json is missing after head exists", function() + local dir = os.tmpname() + os.remove(dir) + + local cfg = fake_cfg() + cfg.state_dir = dir + local first, first_err = main_mod.run_init(cfg, { machine = fake_machine("{}") }) + assert(first, first_err) + + assert(os.remove(dir .. "/config.json")) + + local second, second_err = main_mod.run_init(cfg, { machine = fake_machine("{}") }) + assert_eq(second, nil) + assert(tostring(second_err):find("config.json", 1, true), tostring(second_err)) + assert(tostring(second_err):find("wipe state_dir", 1, true), tostring(second_err)) +end) + +test("init fails when checkpoint snapshot is missing after head exists", function() + local dir = os.tmpname() + os.remove(dir) + + local cfg = fake_cfg() + cfg.state_dir = dir + local first, first_err = main_mod.run_init(cfg, { machine = fake_machine("{}") }) + assert(first, first_err) + + local loaded = assert(checkpoint.load(dir)) + assert(os.execute("rm -rf '" .. loaded.snapshot_dir:gsub("'", "'\\''") .. "'")) + + local second, second_err = main_mod.run_init(cfg, { machine = fake_machine("{}") }) + assert_eq(second, nil) + assert(tostring(second_err):find("snapshot", 1, true), tostring(second_err)) + assert(tostring(second_err):find("wipe state_dir", 1, true), tostring(second_err)) +end) + test("runner happy path replays inputs and writes checkpoint", function() local checkpoint_writes = {} local checkpoint_mod = {