Support hildr

config.py

@@ -290,6 +290,24 @@ def __init__(self, paths: OPPaths = None):
         This is also passed to the devnet L1 (if started) currently, but unclear if it's needed.
         """
 
+        self.l2_hildr_engine_rpc = "http://127.0.0.1:10545"
+        """
+        Protocol + address + port to use to connect to the L2 RPC server attached to the hildr execution
+        engine ("http://127.0.0.1:10545" by default).
+        """
+
+        self.l2_hildr_engine_authrpc = "http://127.0.0.1:10551"
+        """
+        Protocol + address + port to use to connect to the authenticated RPC (authrpc) server
+        attached to the hildr execution engine, which serves the hildr engine API ("http://127.0.0.1:10551" by
+        default).
+        """
+
+        self.l2_hildr_node_rpc = "http://127.0.0.1:11545"
+        """
+        Address to use to connect to the hildr-node RPC server ("http://127.0.0.1:11545" by default).
+        """
+
         self.deployments = None
         """
         Dictionary containing a mapping from rollup contract names to the address at which they're
@@ -447,6 +465,86 @@ def __init__(self, paths: OPPaths = None):
         Ignored if :py:attribute:`node_metrics` is False.
         """
 
+        # ==========================================================================================
+        # L2 Hildr Execution Engine Configuration
+
+        self.l2_hildr_engine_data_dir = os.path.join(self.db_path, "l2_hildr_engine")
+        """Geth data directory for the L2 hildr engine."""
+
+        # See also the properties starting with `l2_engine` below which are paths derived from
+        # :py:attribute:`l2_hildr_engine_data_dir`.
+
+        self.l2_hildr_chain_id = 42069
+        """Chain ID of the local L2 hildr."""
+
+        self.l2_hildr_engine_verbosity = 3
+        """Geth verbosity level (from 0 to 5, see op-geth --help)."""
+
+        self.l2_hildr_engine_p2p_port = 40313
+        """Port to use for the p2p server of the L2 hildr engine (30313 by default)."""
+
+        self.l2_hildr_engine_rpc_listen_addr = "0.0.0.0"
+        """Address the L2 hildr engine http RPC server should bind to ("0.0.0.0" by default)."""
+
+        self.l2_hildr_engine_rpc_listen_port = 10545
+        """Port to use for the L2 hildr engine http JSON-RPC server."""
+
+        self.l2_hildr_engine_rpc_ws_listen_addr = "0.0.0.0"
+        """Address the L2 hildr engine WebSocket RPC server should bind to ("0.0.0.0" by default)."""
+
+        self.l2_hildr_engine_rpc_ws_listen_port = 10546
+        """Port to use for the WebSocket JSON_RPC server."""
+
+        self.l2_hildr_engine_authrpc_listen_addr = "0.0.0.0"
+        """Address the L2 hildr engine authRPC server should bind to ("0.0.0.0" by default)."""
+
+        self.l2_hildr_engine_authrpc_listen_port = 10551
+        """Port to use for the L2 hildr engine authRPC server (9551 by default)."""
+
+        self.l2_hildr_engine_history_transactions = 2350000
+        """
+        Number of recent blocks to maintain transactions index for (default = about one
+        year (geth default), 0 = entire chain)
+
+        This is the `--txlookuplimit` option in geth <= 1.12 and `--history.transactions` in geth >=
+        1.13.
+        """
+
+        self.l2_hildr_engine_disable_tx_gossip = True
+        """
+        Whether to disable transaction pool gossiping (True by default).
+
+        In a system with a single sequencer, publicizing the mempool holds very little advantage:
+        it can cause spam by MEV searchers trying to frontrun or backrun transactions.
+        On the flip side, if the node crashes, gossiping can help refill the sequencer's mempool.
+
+        I believe it's possible to set this to False (enable gossip) but restrict the peers in
+        another way, such that "centralized redundancy" (multiple nodes ran by the same entity)
+        can be achieved.
+
+        This is currently pretty irrelevant, because we hardcode the --maxpeers=0 and --nodiscover
+        flags.
+        """
+
+        # === Metrics ===
+
+        self.l2_hildr_engine_metrics = False
+        """
+        Whether to record metrics in the L2 hildr engine (False by default).
+        """
+
+        self.l2_hildr_engine_metrics_listen_port = 10060
+        """
+        Port to the L2 hildr engine metrics server should bind to (8060 by default).
+        Ignored if :py:attribute:`node_metrics` is False.
+        """
+
+        self.l2_hildr_engine_metrics_listen_addr = "0.0.0.0"
+        """
+        Address the L2 hildr engine metrics server should bind to ("0.0.0.0" by default).
+        Ignored if :py:attribute:`node_metrics` is False.
+        """
+
         # ==========================================================================================
         # Node Configuration
 
@@ -538,6 +636,53 @@ def __init__(self, paths: OPPaths = None):
         Ignored if :py:attribute:`node_metrics` is False.
         """
 
+        # ==========================================================================================
+        # Hildr Service Start Flag
+        self.l2_hildr_enabled = False
+        """
+        Whether to enable Hildr service (hildr node + hildr engine — False by default).
+        """
+
+        # Hildr Node Configuration
+
+        # === RPC ===
+        self.l2_hildr_node_rpc_listen_addr = "0.0.0.0"
+        """
+        Address the node RPC server should bind to ("0.0.0.0" by default).
+
+        Used for the "optimism" namespace API
+        (https://community.optimism.io/docs/developers/build/json-rpc/) and the "admin" namespace
+        (cf. :py:attribute:`node_enable_admin`).
+        """
+
+        self.l2_hildr_node_rpc_listen_port = 11545
+        """
+        Port the hildr node RPC server should bind to (11545 by default).
+
+        Used for the "optimism" namespace API
+        (https://community.optimism.io/docs/developers/build/json-rpc/) and the "admin" namespace
+        (cf. :py:attribute:`node_enable_admin`).
+        """
+
+        # === Metrics ===
+
+        self.l2_hildr_node_metrics = False
+        """
+        Whether to record metrics in the L2 node (False by default).
+        """
+
+        self.l2_hildr_node_metrics_listen_port = 11300
+        """
+        Port to the l2 node metrics server should bind to (7300 by default).
+        Ignored if :py:attribute:`node_metrics` is False.
+        """
+
+        self.l2_hildr_node_metrics_listen_addr = "0.0.0.0"
+        """
+        Address the L2 node metrics server should bind to ("0.0.0.0" by default).
+        Ignored if :py:attribute:`node_metrics` is False.
+        """
+
         # ==========================================================================================
         # Proposer Configuration
 
@@ -751,6 +896,11 @@ def l2_engine_chaindata_dir(self):
         """Directory storing chain data for the L2 engine."""
         return os.path.join(self.l2_engine_data_dir, "geth", "chaindata")
 
+    @property
+    def l2_hildr_engine_chaindata_dir(self):
+        """Directory storing chain data for the L2 engine."""
+        return os.path.join(self.l2_hildr_engine_data_dir, "geth", "chaindata")
+
     # ==============================================================================================
 
     def validate(self):
@@ -822,6 +972,10 @@ def use_op_doc_config(self):
         self.l2_engine_authrpc = "http://127.0.0.1:8551"
         self.l2_node_rpc = "http://127.0.0.1:8547"
 
+        self.l2_hildr_engine_rpc = "http://127.0.0.1:10545"
+        self.l2_hildr_engine_authrpc = "http://127.0.0.1:10551"
+        self.l2_hildr_node_rpc = "http://127.0.0.1:11545"
+
         self.jwt_secret_path = "jwt.txt"
 
         # === Devnet L1 ===

deps.py

@@ -468,4 +468,58 @@ def install_geth():
 
     print(f"Successfully installed geth {INSTALL_GETH_VERSION} as ./bin/geth")
 
+
+####################################################################################################
+
+JDK_MIN_VERSION = "21"
+"""Version of Jdk to install if not found."""
+
+JDK_MAX_VERSION = "22"
+"""Maximum JDK version found to work."""
+
+JDK_INSTALL_VERSION = "21.0.1"
+"""Version of JDK to install if not found."""
+
+# --------------------------------------------------------------------------------------------------
+
+
+def check_or_install_jdk():
+    """
+    Check if JDK is installed and is the correct version, otherwise prompts the user to install
+    it via SDKMAN.
+    """
+
+    # Check if Node is installed and is the correct version.
+    if shutil.which("java") is not None:
+        version = lib.run("get jdk version", "java -version").replace("version", "").replace("\"", "")
+        version = re.search(r"((java|openjdk)( +))(\d+(\.\d+)+)", version)
+        version = "0" if version is None else version.group(4)
+        print(f"jdk version: {version}")
+        if version >= JDK_INSTALL_VERSION:
+            return
+
+    def sdk_install_jdk():
+        lib.run(f"install JDK {JDK_INSTALL_VERSION}",
+                f"bash -c '. ~/.sdkman/bin/sdkman-init.sh; echo Y | sdk install java {JDK_INSTALL_VERSION}-amzn'")
+        print(f"Successfully installed JDK {JDK_INSTALL_VERSION}")
+
+    if os.path.isfile(os.path.expanduser("~/.sdkman/bin/sdkman-init.sh")):
+        # We have SDKMAN, try using required version or installing it.
+        try:
+            lib.run(f"init sdkman", f"bash -c '. ~/.sdkman/bin/sdkman-init.sh; sdk default java {JDK_INSTALL_VERSION}-amzn'")
+        except Exception:
+            if lib.ask_yes_no(f"JDK {JDK_INSTALL_VERSION} is required. SDKMAN is installed. "
+                              f"Install with SDKMAN?"):
+                sdk_install_jdk()
+            else:
+                raise Exception(f"JDK {JDK_INSTALL_VERSION} is required.")
+    else:
+        # Install SDKMAN + JDK.
+        sdkman_url = f"https://get.sdkman.io"
+        if lib.ask_yes_no(f"JDK {JDK_INSTALL_VERSION} is required. Install SDKMAN + JDK?"):
+            lib.run("install sdkman", f"curl -s {sdkman_url} | bash")
+            sdk_install_jdk()
+        else:
+            raise Exception(f"JDK {JDK_INSTALL_VERSION} is required.")
+
 ####################################################################################################

hildr_engine.py

@@ -0,0 +1,114 @@
+import os
+import shutil
+import sys
+
+from config import Config
+from processes import PROCESS_MGR
+
+import libroll as lib
+
+
+####################################################################################################
+
+def start(config: Config):
+    """
+    Spin the L2 hildr execution engine (op-geth), then wait for it to be ready.
+    """
+
+    lib.ensure_port_unoccupied(
+        "op-geth", config.l2_hildr_engine_rpc_listen_addr, config.l2_hildr_engine_rpc_listen_port)
+
+    # Create geth db if it doesn't exist.
+    os.makedirs(config.l2_hildr_engine_data_dir, exist_ok=True)
+
+    if not os.path.exists(config.l2_hildr_engine_chaindata_dir):
+        log_file = "logs/init_l2_hildr_genesis.log"
+        print(f"Directory {config.l2_hildr_engine_chaindata_dir} missing, "
+              "importing genesis in op-geth node."
+              f"Logging to {log_file}")
+        lib.run(
+            "initializing genesis",
+            ["op-geth",
+             f"--verbosity={config.l2_hildr_engine_verbosity}",
+             "init",
+             f"--datadir={config.l2_hildr_engine_data_dir}",
+             config.paths.l2_genesis_path])
+
+    log_file_path = "logs/l2_hildr_engine.log"
+    print(f"Starting op-geth node for hildr. Logging to {log_file_path}")
+    sys.stdout.flush()
+
+    log_file = open(log_file_path, "w")
+
+    PROCESS_MGR.start(
+        "starting op-geth",
+        [
+            "op-geth",
+
+            f"--datadir={config.l2_hildr_engine_data_dir}",
+            f"--verbosity={config.l2_hildr_engine_verbosity}",
+
+            f"--networkid={config.l2_hildr_chain_id}",
+            "--syncmode=full",  # doesn't matter, it's only us
+            "--gcmode=archive",
+
+            # No peers: the blockchain is only this node
+            "--nodiscover",
+            "--maxpeers=0",
+
+            # p2p network config, avoid conflicts with L1 geth nodes
+            f"--port={config.l2_hildr_engine_p2p_port}",
+
+            "--rpc.allow-unprotected-txs",  # allow legacy transactions for deterministic deployment
+
+            # HTTP JSON-RPC server config
+            "--http",
+            "--http.corsdomain=*",
+            "--http.vhosts=*",
+            f"--http.addr={config.l2_hildr_engine_rpc_listen_addr}",
+            f"--http.port={config.l2_hildr_engine_rpc_listen_port}",
+            "--http.api=web3,debug,eth,txpool,net,engine",
+
+            # WebSocket JSON-RPC server config
+            "--ws",
+            f"--ws.addr={config.l2_hildr_engine_rpc_ws_listen_addr}",
+            f"--ws.port={config.l2_hildr_engine_rpc_ws_listen_port}",
+            "--ws.origins=*",
+            "--ws.api=debug,eth,txpool,net,engine",
+
+            # Authenticated RPC config
+            f"--authrpc.addr={config.l2_hildr_engine_authrpc_listen_addr}",
+            f"--authrpc.port={config.l2_hildr_engine_authrpc_listen_port}",
+            "--authrpc.vhosts=*",
+            f"--authrpc.jwtsecret={config.jwt_secret_path}",
+
+            # Metrics Options
+            *([] if not config.l2_hildr_engine_metrics else [
+                "--metrics",
+                f"--metrics.port={config.l2_hildr_engine_metrics_listen_port}",
+                f"--metrics.addr={config.l2_hildr_engine_metrics_listen_addr}"]),
+
+            # Configuration for the rollup engine
+            f"--rollup.disabletxpoolgossip={config.l2_hildr_engine_disable_tx_gossip}",
+
+            # Other geth options
+            f"--txlookuplimit={config.l2_hildr_engine_history_transactions}",
+
+        ], forward="fd", stdout=log_file)
+
+    lib.wait_for_rpc_server("127.0.0.1", config.l2_hildr_engine_rpc_listen_port)
+
+
+####################################################################################################
+
+def clean(config: Config):
+    """
+    Cleans up L2 execution engine's databases, such that trying to start the L2 execution engine
+    (op-geth) will proceed as though it had never been started before (this might cause problems
+    if the rest of the system hasn't been similarly reset).
+    """
+    if os.path.exists(config.l2_hildr_engine_data_dir):
+        print(f"Cleaning up {config.l2_hildr_engine_data_dir}")
+        shutil.rmtree(config.l2_hildr_engine_data_dir, ignore_errors=True)
+
+####################################################################################################