offline tx retrieval tool/library WIP

BUILD_AND_TEST.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+
+# Build and test gettx tool
+#set -e
+
+# Ensure we're in the git repository root
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+cd "$SCRIPT_DIR"
+
+# Initialize submodules and build directory on first run
+if [[ ! -d "build" ]]; then
+  echo "=== Initializing git submodules ===" >> /tmp/gettx_test_output.txt
+  git submodule update --init --recursive >> /tmp/gettx_test_output.txt 2>&1
+  if [[ "$?" != "0" ]]; then
+    echo "*** Git submodule initialization failed"
+    exit 1
+  fi
+  echo "=== Creating build directory ===" >> /tmp/gettx_test_output.txt
+  mkdir build
+fi
+
+cd build
+
+# Run cmake configuration if needed
+if [[ ! -f "CMakeCache.txt" ]]; then
+  echo "=== Running cmake configuration ===" >> /tmp/gettx_test_output.txt
+  cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=clang-21 -DCMAKE_CXX_COMPILER=clang++-21 >> /tmp/gettx_test_output.txt 2>&1
+  if [[ "$?" != "0" ]]; then
+    tail --l 30 /tmp/gettx_test_output.txt
+    echo "*** Cmake configuration failed, details available at /tmp/gettx_test_output.txt"
+    exit 1
+  fi
+fi
+
+echo "=== Building gettx ===" > /tmp/gettx_test_output.txt
+cmake --build . --target gettx -j8 >> /tmp/gettx_test_output.txt 2>&1
+if [[ "$?" != "0" ]]; then
+  tail --l 30 /tmp/gettx_test_output.txt
+  echo "*** Building gettx failed, details available at /tmp/gettx_test_output.txt"
+  exit 1
+fi
+
+echo "" >> /tmp/gettx_test_output.txt
+echo "=== Running gettx test ===" >> /tmp/gettx_test_output.txt
+# Testing with transaction from TRANSACTION_EXAMPLES.txt:
+# Transaction: FYLEbPNbsXAiXZNjVSRERJ0/zybCh+Z3JTbH97K/gAg=
+# lt: 2000001, workchain: -1
+# address: Ef8zMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM0vF (base64)
+./tools/gettx/gettx tx \
+  --workchain -1 \
+  --address Ef80UXx731GHxVr0-LYf3DIViMerdo3uJLAG3ykQZFjXz2kW \
+  --lt 2000001 \
+  --hash 69XTCXdLrEOtYJB76hrAi5rx2fJ80kFl8MwefrLjNYU= \
+  --count 1 \
+  --db-path ../db \
+  --format json >> /tmp/gettx_test_output.txt 2>&1
+if [[ "$?" != "0" ]]; then
+  tail --l 30 /tmp/gettx_test_output.txt
+  echo "*** Running gettx tx failed, details available at /tmp/gettx_test_output.txt"
+  exit 1
+fi
+tail --l 30 /tmp/gettx_test_output.txt
+echo "*** Running gettx tx test finished successfully."
+
+echo "" >> /tmp/gettx_test_output.txt
+echo "=== Testing gettx block subcommand ===" >> /tmp/gettx_test_output.txt
+# Test block retrieval - get masterchain block 2 and all its shard transactions
+./tools/gettx/gettx block \
+  --seqno 2 \
+  --db-path ../db >> /tmp/gettx_test_output.txt 2>&1
+if [[ "$?" != "0" ]]; then
+  tail --l 30 /tmp/gettx_test_output.txt
+  echo "*** Running gettx block failed, details available at /tmp/gettx_test_output.txt"
+  exit 1
+fi
+tail --l 30 /tmp/gettx_test_output.txt
+echo "*** Running gettx block test finished successfully."

CMakeLists.txt

@@ -566,6 +566,7 @@ add_subdirectory(crypto)
 add_subdirectory(lite-client)
 add_subdirectory(emulator)
 add_subdirectory(tolk)
+add_subdirectory(tools/gettx)
 
 #BEGIN tonlib
 add_subdirectory(tonlib)

README.md

@@ -1,3 +1,10 @@
+## Offline TX Retrieval Tool
+
+See tools/ directory.
+Includes a C CLI tool and Python wrapper.
+
+<hr/>
+
 <div align="center">
   <a href="https://ton.org">
     <picture>

tools/gettx/CMakeLists.txt

@@ -0,0 +1,91 @@
+# Standalone getTransactions tool for TON validator databases
+
+add_executable(gettx
+  main.cpp
+  db-index.cpp
+  package-reader.cpp
+  transaction-lookup.cpp
+  shard-iterator.cpp
+  block-tx-extractor.cpp
+)
+
+target_include_directories(gettx
+  PRIVATE
+    ${CMAKE_SOURCE_DIR}
+    ${CMAKE_SOURCE_DIR}/validator
+    ${CMAKE_SOURCE_DIR}/crypto
+    ${CMAKE_SOURCE_DIR}/crypto/vm
+    ${CMAKE_SOURCE_DIR}/crypto/block
+    ${CMAKE_SOURCE_DIR}/crypto/tl
+    ${CMAKE_SOURCE_DIR}/tdutils
+    ${CMAKE_SOURCE_DIR}/tdutils/tl
+    ${CMAKE_SOURCE_DIR}/tdutils/db
+)
+
+target_link_libraries(gettx
+  PRIVATE
+    tdutils
+    tddb
+    ton_crypto
+    tl_api
+    tl-utils
+    ton_db
+    ton_block
+    validator-db
+)
+
+target_compile_features(gettx PRIVATE cxx_std_20)
+
+# Shared library for Python bindings
+add_library(gettx_shared SHARED
+  gettx_api.cpp
+  db-index.cpp
+  package-reader.cpp
+  transaction-lookup.cpp
+  shard-iterator.cpp
+  block-tx-extractor.cpp
+  ${CMAKE_SOURCE_DIR}/tl/generate/auto/tl/lite_api.cpp
+)
+
+target_include_directories(gettx_shared
+  PRIVATE
+    ${CMAKE_SOURCE_DIR}
+    ${CMAKE_SOURCE_DIR}/validator
+    ${CMAKE_SOURCE_DIR}/crypto
+    ${CMAKE_SOURCE_DIR}/crypto/vm
+    ${CMAKE_SOURCE_DIR}/crypto/block
+    ${CMAKE_SOURCE_DIR}/crypto/tl
+    ${CMAKE_SOURCE_DIR}/tl
+    ${CMAKE_SOURCE_DIR}/tl/generate
+    ${CMAKE_SOURCE_DIR}/tdutils
+    ${CMAKE_SOURCE_DIR}/tdutils/tl
+    ${CMAKE_SOURCE_DIR}/tdutils/db
+)
+
+# Link libraries normally - don't use whole-archive which can cause issues
+# with undefined symbols from TL objects
+target_link_libraries(gettx_shared
+  PRIVATE
+    tdutils
+    tddb
+    ton_crypto
+    tl_api
+    tl-utils
+    ton_db
+    ton_block
+    validator-db
+)
+
+target_compile_features(gettx_shared PRIVATE cxx_std_20)
+
+# Set visibility to export symbols
+target_compile_options(gettx_shared PRIVATE -fPIC)
+target_compile_definitions(gettx_shared PRIVATE GETTX_EXPORTS)
+
+# Set output name for the shared library
+set_target_properties(gettx_shared PROPERTIES
+  OUTPUT_NAME "gettx"
+  VERSION 1.0.0
+  SOVERSION 1
+  POSITION_INDEPENDENT_CODE ON
+)

tools/gettx/README.md

@@ -0,0 +1,25 @@
+# gettx - Standalone TON Transaction Lookup Tool
+
+A standalone C++/Python tool that provides `liteserver.getTransactions()` functionality by directly reading TON validator database files.
+
+## Building
+
+```bash
+cd /path/to/ton
+mkdir build && cd build
+cmake ..
+cmake --build . --target gettx
+```
+
+## Usage
+
+```bash
+./gettx \
+  --workchain <w> \
+  --address <addr> \
+  --lt <logical_time> \
+  --hash <tx_hash> \
+  --count <n> \
+  --db-path <path> \
+  --format <json|tl>
+```

tools/gettx/block-tx-extractor.cpp

@@ -0,0 +1,143 @@
+#include "block-tx-extractor.h"
+#include "crypto/block/block-auto.h"
+#include "crypto/block/block-parse.h"
+#include "vm/boc.h"
+#include "vm/cellslice.h"
+
+namespace ton {
+
+namespace gettx {
+
+td::Result<std::vector<BlockTransactionExtractor::TransactionInfo>> BlockTransactionExtractor::extract_all_transactions(
+    const ton::BlockIdExt& block_id,
+    const td::BufferSlice& block_data) {
+  std::cerr << "DEBUG: extract_all_transactions called for block " << block_id.to_str()
+            << ", data size=" << block_data.size() << "\n";
+
+  // Deserialize the block BOC
+  auto block_root_result = vm::std_boc_deserialize(block_data.as_slice());
+  if (block_root_result.is_error()) {
+    return td::Status::Error("Failed to deserialize block BOC");
+  }
+
+  auto block_root = block_root_result.move_as_ok();
+  std::cerr << "DEBUG: Block BOC deserialized successfully\n";
+
+  return parse_transactions_from_block(block_id, block_root);
+}
+
+td::Result<std::vector<BlockTransactionExtractor::TransactionInfo>> BlockTransactionExtractor::parse_transactions_from_block(
+    const ton::BlockIdExt& block_id,
+    const vm::Ref<vm::Cell>& block_root) {
+  std::vector<TransactionInfo> transactions;
+
+  // Unpack the block structure: Block -> BlockExtra -> account_blocks
+  block::gen::Block::Record blk;
+  block::gen::BlockExtra::Record extra;
+
+  if (!tlb::unpack_cell(block_root, blk)) {
+    return td::Status::Error("Cannot unpack block header");
+  }
+
+  if (!tlb::unpack_cell(blk.extra, extra)) {
+    return td::Status::Error("Cannot unpack block extra");
+  }
+
+  std::cerr << "DEBUG: Unpacked block extra, account_blocks root exists=" << extra.account_blocks.not_null() << "\n";
+
+  // Create augmented dictionary from account_blocks
+  vm::AugmentedDictionary acc_dict{vm::load_cell_slice_ref(extra.account_blocks), 256,
+                                     block::tlb::aug_ShardAccountBlocks};
+
+  // Iterate through all accounts in the account_blocks dictionary
+  int account_count = 0;
+  int tx_count = 0;
+
+  for (auto acc_iter = acc_dict.begin(); acc_iter != acc_dict.end(); ++acc_iter) {
+    account_count++;
+    auto acc_entry = *acc_iter;
+
+    // Parse AccountBlock
+    block::gen::AccountBlock::Record acc_blk;
+    if (!tlb::csr_unpack(std::move(acc_entry.second), acc_blk)) {
+      std::cerr << "DEBUG: Warning: Failed to unpack AccountBlock\n";
+      continue;
+    }
+
+    ton::StdSmcAddress account_addr = acc_blk.account_addr;
+    std::cerr << "DEBUG: Processing account " << account_addr.to_hex()
+              << " with transactions dict\n";
+
+    // Create dictionary from transactions (64-bit keys = logical times)
+    vm::AugmentedDictionary trans_dict{vm::DictNonEmpty(), std::move(acc_blk.transactions), 64,
+                                        block::tlb::aug_AccountTransactions};
+
+    // Iterate through all transactions for this account
+    for (auto tx_iter = trans_dict.begin(); tx_iter != trans_dict.end(); ++tx_iter) {
+      auto tx_entry = *tx_iter;
+
+      // Extract LT from key
+      ton::LogicalTime lt = tx_entry.first.get_int(64);
+
+      // Get transaction cell
+      auto tx_csr = tx_entry.second;
+      if (tx_csr.is_null() || tx_csr->size_refs() == 0) {
+        continue;
+      }
+
+      vm::Ref<vm::Cell> tx_root = tx_csr->prefetch_ref();
+      if (tx_root.is_null()) {
+        continue;
+      }
+
+      // Parse transaction to get metadata
+      block::gen::Transaction::Record trans;
+      if (!tlb::unpack_cell(tx_root, trans)) {
+        std::cerr << "DEBUG: Warning: Failed to unpack transaction LT=" << lt << "\n";
+        continue;
+      }
+
+      // Calculate transaction hash
+      ton::Bits256 tx_hash;
+      tx_hash.as_slice().copy_from(tx_root->get_hash().as_slice());
+
+      // Extract total_fees
+      td::uint64 total_fees = 0;
+      if (trans.total_fees.not_null()) {
+        td::RefInt256 fee_value;
+        td::Ref<vm::Cell> fee_extra;
+        if (block::unpack_CurrencyCollection(trans.total_fees, fee_value, fee_extra)) {
+          if (fee_value.not_null()) {
+            unsigned char bytes[32];
+            fee_value->export_bytes(bytes, 32, false);
+            // Little-endian conversion from bytes 0-7 to uint64
+            for (int i = 0; i < 8 && i < 32; i++) {
+              total_fees |= ((td::uint64)bytes[i]) << (8 * i);
+            }
+          }
+        }
+      }
+
+      // Create transaction info
+      TransactionInfo info;
+      info.workchain = block_id.id.workchain;
+      info.account_addr = account_addr;
+      info.lt = lt;
+      info.hash = tx_hash;
+      info.utime = trans.now;
+      info.total_fees = total_fees;
+      info.root = tx_root;
+      info.block_id = block_id;
+
+      transactions.push_back(std::move(info));
+      tx_count++;
+    }
+  }
+
+  std::cerr << "DEBUG: Extracted " << tx_count << " transactions from " << account_count << " accounts\n";
+  return transactions;
+}
+
+}  // namespace gettx
+
+}  // namespace ton