Address user-reported secret leak, RPC proxy retry, and watchdog container issues

README.md

@@ -98,7 +98,7 @@ nonce on demand (keyed operator tool).
 `setup` requires: `CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT`, `CARTESI_SEQUENCER_BLOCKCHAIN_ID`, `CARTESI_SEQUENCER_APP_ADDRESS`, `CARTESI_SEQUENCER_BATCH_SUBMITTER_ADDRESS`.
 `run` requires: `CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT`, `CARTESI_SEQUENCER_AUTH_PRIVATE_KEY` (or `_FILE`); it refuses to boot until `setup` has completed.
 
-Optional: `CARTESI_SEQUENCER_HTTP_ADDR` (default `127.0.0.1:3000`, `run`), `CARTESI_SEQUENCER_DATA_DIR` (default `sequencer-data` — SQLite file is `sequencer.db` inside; created if missing), `CARTESI_SEQUENCER_PREEMPTIVE_MARGIN_BLOCKS` (default `300`), `CARTESI_SEQUENCER_SECONDS_PER_BLOCK` (default `12`), `CARTESI_SEQUENCER_L1_READ_STALE_AFTER_BLOCKS` (default `600`), `CARTESI_SEQUENCER_LONG_BLOCK_RANGE_ERROR_CODES` (default `-32005,-32600,-32602,-32616`), `CARTESI_SEQUENCER_AUTH_PRIVATE_KEY_FILE` (alternative to `CARTESI_SEQUENCER_AUTH_PRIVATE_KEY`; first line of the file is the key), `CARTESI_SEQUENCER_BATCH_SUBMITTER_IDLE_POLL_INTERVAL_MS`, `CARTESI_SEQUENCER_BATCH_SUBMITTER_CONFIRMATION_DEPTH`.
+Optional: `CARTESI_SEQUENCER_HTTP_ADDR` (default `127.0.0.1:3000`, `run`), `CARTESI_SEQUENCER_DATA_DIR` (default `sequencer-data` — SQLite file is `sequencer.db` inside; created if missing), `CARTESI_SEQUENCER_PREEMPTIVE_MARGIN_BLOCKS` (default `300`), `CARTESI_SEQUENCER_SECONDS_PER_BLOCK` (default `12`), `CARTESI_SEQUENCER_L1_READ_STALE_AFTER_BLOCKS` (default `600`), `CARTESI_SEQUENCER_LONG_BLOCK_RANGE_ERROR_CODES` (default `-32005,-32012,-32600,-32602,-32616`), `CARTESI_SEQUENCER_AUTH_PRIVATE_KEY_FILE` (alternative to `CARTESI_SEQUENCER_AUTH_PRIVATE_KEY`; first line of the file is the key), `CARTESI_SEQUENCER_BATCH_SUBMITTER_IDLE_POLL_INTERVAL_MS`, `CARTESI_SEQUENCER_BATCH_SUBMITTER_CONFIRMATION_DEPTH`.
 
 By default the blockchain endpoint must be `https://` unless its host is loopback (`localhost`, `127.0.0.0/8`, `::1`) — a guard against accidentally sending L1 traffic to a public RPC in the clear. Set `CARTESI_SEQUENCER_ALLOW_INSECURE_RPC=true` (or `--allow-insecure-rpc`) to permit plaintext `http://` to a non-loopback host on a **trusted private network** — e.g. a Docker Compose / Kubernetes service name (`http://anvil:8545`), `host.docker.internal`, or a private-VPC IP.
 

sequencer/src/l1/partition.rs

@@ -8,8 +8,22 @@
 //! This module is stateless: callers pass the retry error codes explicitly. There is no
 //! global mutable state; `RunConfig` owns the codes and passes them down via configs.
 
-/// Default RPC error codes that trigger partition retry (e.g. Infura -32005, Alchemy -32600/-32602, QuickNode -32616).
-pub const DEFAULT_LONG_BLOCK_RANGE_ERROR_CODES: &[&str] = &["-32005", "-32600", "-32602", "-32616"];
+/// Default RPC error codes that trigger partition retry (e.g. Infura -32005,
+/// Alchemy -32600/-32602, QuickNode -32616, Goldsky / some JSON-RPC proxies
+/// -32012).
+///
+/// **Overloaded codes (accepted).** `-32005` and `-32012` are not range-only
+/// across providers: Alloy's transport also treats bare `-32005` as Infura
+/// rate-limit, and `-32012` + `"credits"` as QuickNode credit rate-limit.
+/// We still match on the bare code — the same policy already used for
+/// `-32005` — because range-error messages are not stable across proxies
+/// (Goldsky vs custom gateways). A persistent QuickNode credits limit that
+/// survives Alloy's own retries can therefore fan an N-block scan into
+/// `2N−1` partition queries before failing; operators on QuickNode can drop
+/// `-32012` from `CARTESI_SEQUENCER_LONG_BLOCK_RANGE_ERROR_CODES` if that
+/// matters. We deliberately do **not** add message-marker heuristics here.
+pub const DEFAULT_LONG_BLOCK_RANGE_ERROR_CODES: &[&str] =
+    &["-32005", "-32012", "-32600", "-32602", "-32616"];
 
 use alloy::contract::Error as ContractError;
 use alloy::contract::Event;
@@ -373,6 +387,27 @@ mod tests {
         assert!(!error_message_matches_retry_codes("ok", &[]));
     }
 
+    #[test]
+    fn default_codes_include_proxy_range_exceeded() {
+        assert!(
+            DEFAULT_LONG_BLOCK_RANGE_ERROR_CODES.contains(&"-32012"),
+            "default codes must include -32012 (JSON-RPC proxy range limit)"
+        );
+    }
+
+    #[test]
+    fn proxy_range_exceeded_error_triggers_retry() {
+        let msg = r#"-32012: getLogs request exceeded max allowed range, data: {"code":"ErrGetLogsExceededMaxAllowedRange"}"#;
+        let codes: Vec<String> = DEFAULT_LONG_BLOCK_RANGE_ERROR_CODES
+            .iter()
+            .map(|c| (*c).to_string())
+            .collect();
+        assert!(
+            error_message_matches_retry_codes(msg, &codes),
+            "proxy -32012 error must match retry codes"
+        );
+    }
+
     #[test]
     fn decode_evm_advance_input_round_trips() {
         let encoded = EvmAdvanceCall {

sequencer/src/runtime/config.rs

@@ -116,13 +116,15 @@ pub struct KeyArgs {
     #[arg(
         long,
         env = "CARTESI_SEQUENCER_AUTH_PRIVATE_KEY",
+        hide_env_values = true,
         group = "batch_submitter_key_source"
     )]
     batch_submitter_private_key: Option<String>,
     /// Path to a file whose first line contains the batch submitter private key.
     #[arg(
         long,
         env = "CARTESI_SEQUENCER_AUTH_PRIVATE_KEY_FILE",
+        hide_env_values = true,
         group = "batch_submitter_key_source"
     )]
     batch_submitter_private_key_file: Option<String>,
@@ -167,10 +169,18 @@ fn resolve_key_source(
 #[derive(Debug, Clone, Args)]
 pub struct OptionalKeyArgs {
     /// Hex-encoded batch-submitter private key.
-    #[arg(long, env = "CARTESI_SEQUENCER_AUTH_PRIVATE_KEY")]
+    #[arg(
+        long,
+        env = "CARTESI_SEQUENCER_AUTH_PRIVATE_KEY",
+        hide_env_values = true
+    )]
     batch_submitter_private_key: Option<String>,
     /// Path to a file whose first line is the batch-submitter private key.
-    #[arg(long, env = "CARTESI_SEQUENCER_AUTH_PRIVATE_KEY_FILE")]
+    #[arg(
+        long,
+        env = "CARTESI_SEQUENCER_AUTH_PRIVATE_KEY_FILE",
+        hide_env_values = true
+    )]
     batch_submitter_private_key_file: Option<String>,
 }
 
@@ -204,7 +214,7 @@ impl OptionalKeyArgs {
 pub struct SetupConfig {
     #[arg(long, env = "CARTESI_SEQUENCER_DATA_DIR", default_value = DEFAULT_DATA_DIR, value_parser = parse_non_empty_string)]
     pub data_dir: String,
-    #[arg(long, env = "CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT", value_parser = parse_non_empty_string)]
+    #[arg(long, env = "CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT", hide_env_values = true, value_parser = parse_non_empty_string)]
     pub eth_rpc_url: String,
     /// Allow plaintext (`http://`) RPC to a non-loopback host — a trusted
     /// private network (Docker/K8s service name, `host.docker.internal`, a
@@ -330,7 +340,7 @@ pub struct RunConfig {
     pub http_addr: String,
     #[arg(long, env = "CARTESI_SEQUENCER_DATA_DIR", default_value = DEFAULT_DATA_DIR, value_parser = parse_non_empty_string)]
     pub data_dir: String,
-    #[arg(long, env = "CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT", value_parser = parse_non_empty_string)]
+    #[arg(long, env = "CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT", hide_env_values = true, value_parser = parse_non_empty_string)]
     pub eth_rpc_url: String,
     /// Allow plaintext (`http://`) RPC to a non-loopback host — a trusted
     /// private network (Docker/K8s service name, `host.docker.internal`, a
@@ -411,7 +421,7 @@ impl RunConfig {
 pub struct FlushConfig {
     #[arg(long, env = "CARTESI_SEQUENCER_DATA_DIR", default_value = DEFAULT_DATA_DIR, value_parser = parse_non_empty_string)]
     pub data_dir: String,
-    #[arg(long, env = "CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT", value_parser = parse_non_empty_string)]
+    #[arg(long, env = "CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT", hide_env_values = true, value_parser = parse_non_empty_string)]
     pub eth_rpc_url: String,
     /// Allow plaintext (`http://`) RPC to a non-loopback host — a trusted
     /// private network (Docker/K8s service name, `host.docker.internal`, a
@@ -664,6 +674,7 @@ mod tests {
             config.long_block_range_error_codes,
             vec![
                 "-32005".to_string(),
+                "-32012".to_string(),
                 "-32600".to_string(),
                 "-32602".to_string(),
                 "-32616".to_string()
@@ -727,6 +738,91 @@ mod tests {
         );
     }
 
+    /// Render `--help` for `subcmd` inside a child process whose env carries
+    /// the given secret values. Mutating this process's env is unsound under
+    /// parallel `cargo test` (and neighbouring Clap parses race on the same
+    /// vars), so the probe runs via `Command::env` on a re-exec of this
+    /// test binary.
+    fn help_text_with_secret_env(subcmd: &str, secrets: &[(&str, &str)]) -> String {
+        let out_path = std::env::temp_dir().join(format!(
+            "sequencer-help-leak-{}-{}-{}.txt",
+            subcmd,
+            std::process::id(),
+            std::time::SystemTime::now()
+                .duration_since(std::time::UNIX_EPOCH)
+                .expect("clock")
+                .as_nanos()
+        ));
+
+        let exe = std::env::current_exe().expect("current_exe");
+        let mut cmd = std::process::Command::new(&exe);
+        cmd.args([
+            "--exact",
+            "runtime::config::tests::help_does_not_leak_private_key_value",
+            "--quiet",
+        ])
+        .env("SEQUENCER_HELP_LEAK_OUT", &out_path)
+        .env("SEQUENCER_HELP_LEAK_SUBCMD", subcmd);
+        for (key, value) in secrets {
+            cmd.env(key, value);
+        }
+
+        let status = cmd.status().expect("spawn help-leak child");
+        assert!(
+            status.success(),
+            "help-leak child failed for subcommand {subcmd}: {status}"
+        );
+        let help = std::fs::read_to_string(&out_path)
+            .unwrap_or_else(|err| panic!("read help output {}: {err}", out_path.display()));
+        let _ = std::fs::remove_file(&out_path);
+        help
+    }
+
+    #[test]
+    fn help_does_not_leak_private_key_value() {
+        // Child arm: render help under the caller's env and write it out.
+        // Exits the process so we never nest another spawn.
+        if let Ok(out_path) = std::env::var("SEQUENCER_HELP_LEAK_OUT") {
+            let subcmd = std::env::var("SEQUENCER_HELP_LEAK_SUBCMD")
+                .expect("SEQUENCER_HELP_LEAK_SUBCMD must be set in the child");
+            let help = Cli::try_parse_from(["sequencer", subcmd.as_str(), "--help"])
+                .expect_err("--help must short-circuit as a clap error")
+                .to_string();
+            std::fs::write(&out_path, help).expect("write help output");
+            std::process::exit(0);
+        }
+
+        let sentinel_key = "0xSENTINEL_PRIVATE_KEY_DEADBEEF1234567890abcdef";
+        let sentinel_file = "/secret/path/key.pem";
+        let sentinel_rpc = "https://eth-mainnet.g.alchemy.com/v2/SECRET_TOKEN_XYZ";
+        let secrets = [
+            ("CARTESI_SEQUENCER_AUTH_PRIVATE_KEY", sentinel_key),
+            ("CARTESI_SEQUENCER_AUTH_PRIVATE_KEY_FILE", sentinel_file),
+            ("CARTESI_SEQUENCER_BLOCKCHAIN_HTTP_ENDPOINT", sentinel_rpc),
+        ];
+
+        for subcmd in ["run", "setup", "flush-mempool"] {
+            let help = help_text_with_secret_env(subcmd, &secrets);
+
+            assert!(
+                !help.contains(sentinel_key),
+                "{subcmd} --help must not print the private key value"
+            );
+            assert!(
+                !help.contains(sentinel_file),
+                "{subcmd} --help must not print the key-file path"
+            );
+            assert!(
+                !help.contains("SECRET_TOKEN_XYZ"),
+                "{subcmd} --help must not print the RPC URL (may contain API tokens)"
+            );
+            assert!(
+                help.contains("CARTESI_SEQUENCER_AUTH_PRIVATE_KEY"),
+                "{subcmd} --help should still mention the env var name"
+            );
+        }
+    }
+
     #[test]
     fn domain_uses_fixed_name_and_version() {
         use alloy_primitives::U256;

tests/fixtures/l1_partition_vector.json

@@ -1,5 +1,5 @@
 {
-  "long_block_range_error_codes": ["-32005", "-32600", "-32602", "-32616"],
+  "long_block_range_error_codes": ["-32005", "-32012", "-32600", "-32602", "-32616"],
   "scenarios": [
     {
       "name": "bisect_on_default_infura_code",
@@ -65,6 +65,20 @@
       ],
       "expect_ok": true
     },
+    {
+      "name": "bisect_on_proxy_range_exceeded_code",
+      "start_block": 100,
+      "end_block": 200,
+      "fail_ranges": [
+        {
+          "from": 100,
+          "to": 200,
+          "message": "-32012: getLogs request exceeded max allowed range"
+        }
+      ],
+      "expect_calls": [[100, 200], [100, 150], [151, 200]],
+      "expect_ok": true
+    },
     {
       "name": "single_block_fail_no_split",
       "start_block": 7,

watchdog/config.lua

@@ -86,7 +86,7 @@ function config.load_init(env)
         retry_attempts = optional_number("CARTESI_WATCHDOG_RETRY_ATTEMPTS", 3, env),
         retry_delay_sec = optional_number("CARTESI_WATCHDOG_RETRY_DELAY_SEC", 5, env),
         long_block_range_error_codes = split_csv(
-            env.CARTESI_WATCHDOG_LONG_BLOCK_RANGE_ERROR_CODES or "-32005,-32600,-32602,-32616"
+            env.CARTESI_WATCHDOG_LONG_BLOCK_RANGE_ERROR_CODES or "-32005,-32012,-32600,-32602,-32616"
         ),
     }
 end

watchdog/l1_reader.lua

@@ -7,8 +7,13 @@ local l1_reader = {}
 
 l1_reader.INPUT_ADDED_TOPIC = "0xc05d337121a6e8605c6ec0b72aa29c4210ffe6e5b9cefdd6a7058188a8f66f98"
 
+-- Bare codes, matching the sequencer's DEFAULT_LONG_BLOCK_RANGE_ERROR_CODES.
+-- `-32005` / `-32012` are provider-overloaded (rate-limit vs range); we accept
+-- that the same way the Rust side does — see the comment on that constant.
+-- Operators can override via CARTESI_WATCHDOG_LONG_BLOCK_RANGE_ERROR_CODES.
 l1_reader.DEFAULT_LONG_BLOCK_RANGE_ERROR_CODES = {
     "-32005",
+    "-32012",
     "-32600",
     "-32602",
     "-32616",

watchdog/sequencer-watchdog

@@ -11,15 +11,43 @@ fi
 lua_root="${CARTESI_WATCHDOG_LUA_ROOT:-/opt/watchdog/lua}"
 lua_bin="${CARTESI_WATCHDOG_LUA_BIN:-lua5.4}"
 
-# Prefer deb-installed cartesi bindings under /usr/lib over stale /usr/local
-# copies left by an older manual install (same ordering as the cartesi-machine
-# wrapper shipped in machine-emulator_*.deb).
+# Deb-installed cartesi bindings under /usr/lib — appended *after* the
+# container's own paths (Dockerfile ENV / operator override) so
+# /opt/watchdog/lib/lcurl.so is found first. Prepending broke lcurl
+# discovery in published images. Lua 5.4 prefers the versioned
+# LUA_{C,}PATH_5_4 variables when set, so derive the effective base from
+# versioned-or-unversioned before composing.
 _cartesi_lua_cpath="/usr/lib/lua/5.4/?.so;/usr/lib/lua/5.4/?/init.so"
 _cartesi_lua_path="/usr/share/lua/5.4/?.lua;/usr/share/lua/5.4/?/init.lua"
-export LUA_CPATH="${_cartesi_lua_cpath};${LUA_CPATH:-;}"
-export LUA_PATH="${_cartesi_lua_path};${LUA_PATH:-;}"
-export LUA_CPATH_5_4="${_cartesi_lua_cpath};${LUA_CPATH_5_4:-;}"
-export LUA_PATH_5_4="${_cartesi_lua_path};${LUA_PATH_5_4:-;}"
+
+# Strip trailing semicolons from $1, then append $2 with an explicit `;`,
+# and finish with `;;` so Lua still expands its defaults at the end.
+# Use `case` (not `[[ == *... ]]`) to trim - a semicolon inside `[[ ]]`
+# is a reserved `;;` token and is easy to mis-quote.
+join_lua_search_path() {
+  local base="$1"
+  local extra="$2"
+  while true; do
+    case "$base" in
+      *";") base="${base%;}" ;;
+      *) break ;;
+    esac
+  done
+  if [ -z "$base" ]; then
+    printf '%s;;' "$extra"
+  else
+    printf '%s;%s;;' "$base" "$extra"
+  fi
+}
+
+_cpath_base="${LUA_CPATH_5_4:-${LUA_CPATH:-}}"
+_path_base="${LUA_PATH_5_4:-${LUA_PATH:-}}"
+_new_cpath="$(join_lua_search_path "${_cpath_base}" "${_cartesi_lua_cpath}")"
+_new_path="$(join_lua_search_path "${_path_base}" "${_cartesi_lua_path}")"
+export LUA_CPATH="${_new_cpath}"
+export LUA_PATH="${_new_path}"
+export LUA_CPATH_5_4="${_new_cpath}"
+export LUA_PATH_5_4="${_new_path}"
 
 cd "${lua_root}"